2024-11-20 06:39:48,395 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 06:39:48,415 main DEBUG Took 0.017870 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 06:39:48,415 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 06:39:48,416 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 06:39:48,417 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 06:39:48,418 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,435 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 06:39:48,452 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,455 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,457 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,457 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,458 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,460 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,460 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,462 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,463 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,464 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,464 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,465 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,465 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,466 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,467 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,469 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:39:48,470 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,470 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 06:39:48,472 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:39:48,474 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 06:39:48,477 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 06:39:48,478 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 06:39:48,480 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 06:39:48,480 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 06:39:48,490 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 06:39:48,493 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 06:39:48,496 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 06:39:48,497 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 06:39:48,497 main DEBUG createAppenders(={Console}) 2024-11-20 06:39:48,498 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-20 06:39:48,499 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 06:39:48,499 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-20 06:39:48,500 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 06:39:48,500 main DEBUG OutputStream closed 2024-11-20 06:39:48,500 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 06:39:48,501 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 06:39:48,501 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-20 06:39:48,601 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 06:39:48,604 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 06:39:48,606 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 06:39:48,608 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 06:39:48,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 06:39:48,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 06:39:48,610 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 06:39:48,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 06:39:48,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 06:39:48,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 06:39:48,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 06:39:48,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 06:39:48,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 06:39:48,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 06:39:48,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 06:39:48,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 06:39:48,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 06:39:48,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 06:39:48,620 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 06:39:48,620 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-20 06:39:48,621 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 06:39:48,622 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-20T06:39:49,011 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c 2024-11-20 06:39:49,016 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 06:39:49,016 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T06:39:49,029 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-20T06:39:49,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T06:39:49,111 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/cluster_a8a03e82-e5a9-0028-3abc-1ed331460361, deleteOnExit=true 2024-11-20T06:39:49,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T06:39:49,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/test.cache.data in system properties and HBase conf 2024-11-20T06:39:49,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T06:39:49,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/hadoop.log.dir in system properties and HBase conf 2024-11-20T06:39:49,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T06:39:49,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T06:39:49,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T06:39:49,249 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T06:39:49,360 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:39:49,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T06:39:49,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T06:39:49,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T06:39:49,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T06:39:49,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T06:39:49,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T06:39:49,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T06:39:49,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T06:39:49,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T06:39:49,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/nfs.dump.dir in system properties and HBase conf 2024-11-20T06:39:49,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/java.io.tmpdir in system properties and HBase conf 2024-11-20T06:39:49,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T06:39:49,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T06:39:49,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T06:39:50,582 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T06:39:50,667 INFO [Time-limited test {}] log.Log(170): Logging initialized @3235ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T06:39:50,771 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T06:39:50,875 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T06:39:50,913 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T06:39:50,914 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T06:39:50,916 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T06:39:50,935 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T06:39:50,941 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/hadoop.log.dir/,AVAILABLE} 2024-11-20T06:39:50,943 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T06:39:51,142 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/java.io.tmpdir/jetty-localhost-32895-hadoop-hdfs-3_4_1-tests_jar-_-any-10660847121462154630/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T06:39:51,158 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:32895} 2024-11-20T06:39:51,159 INFO [Time-limited test {}] server.Server(415): Started @3727ms 2024-11-20T06:39:51,753 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T06:39:51,760 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T06:39:51,761 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T06:39:51,762 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T06:39:51,762 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T06:39:51,762 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/hadoop.log.dir/,AVAILABLE} 2024-11-20T06:39:51,763 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T06:39:51,859 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/java.io.tmpdir/jetty-localhost-33967-hadoop-hdfs-3_4_1-tests_jar-_-any-6030848366761607187/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T06:39:51,860 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:33967} 2024-11-20T06:39:51,860 INFO [Time-limited test {}] server.Server(415): Started @4429ms 2024-11-20T06:39:51,908 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T06:39:52,724 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/cluster_a8a03e82-e5a9-0028-3abc-1ed331460361/data/data1/current/BP-501673584-172.17.0.2-1732084790088/current, will proceed with Du for space computation calculation, 2024-11-20T06:39:52,724 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/cluster_a8a03e82-e5a9-0028-3abc-1ed331460361/data/data2/current/BP-501673584-172.17.0.2-1732084790088/current, will proceed with Du for space computation calculation, 2024-11-20T06:39:52,750 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T06:39:52,792 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2e6a2f963750b75b with lease ID 0x1f16ec729a96d476: Processing first storage report for DS-11a6f10b-a11f-44f5-a877-72f92b360c7e from datanode DatanodeRegistration(127.0.0.1:40437, datanodeUuid=dae0415b-24a9-4464-8e5d-29d94cf63508, infoPort=38513, infoSecurePort=0, ipcPort=46371, storageInfo=lv=-57;cid=testClusterID;nsid=223957858;c=1732084790088) 2024-11-20T06:39:52,794 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2e6a2f963750b75b with lease ID 0x1f16ec729a96d476: from storage DS-11a6f10b-a11f-44f5-a877-72f92b360c7e node DatanodeRegistration(127.0.0.1:40437, datanodeUuid=dae0415b-24a9-4464-8e5d-29d94cf63508, infoPort=38513, infoSecurePort=0, ipcPort=46371, storageInfo=lv=-57;cid=testClusterID;nsid=223957858;c=1732084790088), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-20T06:39:52,795 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2e6a2f963750b75b with lease ID 0x1f16ec729a96d476: Processing first storage report for DS-96ed3f4f-365c-41ea-9f70-227371780ec7 from datanode DatanodeRegistration(127.0.0.1:40437, datanodeUuid=dae0415b-24a9-4464-8e5d-29d94cf63508, infoPort=38513, infoSecurePort=0, ipcPort=46371, storageInfo=lv=-57;cid=testClusterID;nsid=223957858;c=1732084790088) 2024-11-20T06:39:52,795 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2e6a2f963750b75b with lease ID 0x1f16ec729a96d476: from storage DS-96ed3f4f-365c-41ea-9f70-227371780ec7 node DatanodeRegistration(127.0.0.1:40437, datanodeUuid=dae0415b-24a9-4464-8e5d-29d94cf63508, infoPort=38513, infoSecurePort=0, ipcPort=46371, storageInfo=lv=-57;cid=testClusterID;nsid=223957858;c=1732084790088), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T06:39:52,881 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c 2024-11-20T06:39:52,946 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/cluster_a8a03e82-e5a9-0028-3abc-1ed331460361/zookeeper_0, clientPort=52097, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/cluster_a8a03e82-e5a9-0028-3abc-1ed331460361/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/cluster_a8a03e82-e5a9-0028-3abc-1ed331460361/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T06:39:52,957 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52097 2024-11-20T06:39:52,969 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T06:39:52,972 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T06:39:53,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741825_1001 (size=7) 2024-11-20T06:39:53,582 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5 with version=8 2024-11-20T06:39:53,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/hbase-staging 2024-11-20T06:39:53,663 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T06:39:53,934 INFO [Time-limited test {}] client.ConnectionUtils(128): master/61c2066a8a0f:0 server-side Connection retries=45 2024-11-20T06:39:53,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:39:53,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T06:39:53,950 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T06:39:53,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:39:53,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T06:39:54,146 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T06:39:54,224 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T06:39:54,233 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T06:39:54,237 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T06:39:54,262 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1540 (auto-detected) 2024-11-20T06:39:54,263 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T06:39:54,282 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33659 2024-11-20T06:39:54,309 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33659 connecting to ZooKeeper ensemble=127.0.0.1:52097 2024-11-20T06:39:54,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:336590x0, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T06:39:54,412 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33659-0x101584211260000 connected 2024-11-20T06:39:54,480 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T06:39:54,482 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T06:39:54,495 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:39:54,499 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5, hbase.cluster.distributed=false 2024-11-20T06:39:54,529 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T06:39:54,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33659 2024-11-20T06:39:54,536 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33659 2024-11-20T06:39:54,536 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33659 2024-11-20T06:39:54,537 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33659 2024-11-20T06:39:54,537 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33659 2024-11-20T06:39:54,657 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45 2024-11-20T06:39:54,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:39:54,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T06:39:54,660 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T06:39:54,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:39:54,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T06:39:54,664 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T06:39:54,668 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T06:39:54,671 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33847 2024-11-20T06:39:54,674 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33847 connecting to ZooKeeper ensemble=127.0.0.1:52097 2024-11-20T06:39:54,676 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T06:39:54,680 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T06:39:54,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338470x0, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T06:39:54,712 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:338470x0, quorum=127.0.0.1:52097, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:39:54,713 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33847-0x101584211260001 connected 2024-11-20T06:39:54,717 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T06:39:54,725 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T06:39:54,728 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T06:39:54,733 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T06:39:54,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33847 2024-11-20T06:39:54,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33847 2024-11-20T06:39:54,735 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33847 2024-11-20T06:39:54,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33847 2024-11-20T06:39:54,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33847 2024-11-20T06:39:54,752 DEBUG [M:0;61c2066a8a0f:33659 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c2066a8a0f:33659 2024-11-20T06:39:54,753 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c2066a8a0f,33659,1732084793752 2024-11-20T06:39:54,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:39:54,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:39:54,772 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c2066a8a0f,33659,1732084793752 2024-11-20T06:39:54,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T06:39:54,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:54,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:54,795 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T06:39:54,796 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c2066a8a0f,33659,1732084793752 from backup master directory 2024-11-20T06:39:54,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:39:54,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c2066a8a0f,33659,1732084793752 2024-11-20T06:39:54,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:39:54,804 WARN [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T06:39:54,804 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c2066a8a0f,33659,1732084793752 2024-11-20T06:39:54,807 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T06:39:54,809 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T06:39:54,864 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/hbase.id] with ID: f056ae1d-0bc3-461f-a70b-1784272b5485 2024-11-20T06:39:54,864 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/.tmp/hbase.id 2024-11-20T06:39:54,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741826_1002 (size=42) 2024-11-20T06:39:55,279 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/.tmp/hbase.id]:[hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/hbase.id] 2024-11-20T06:39:55,340 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T06:39:55,346 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T06:39:55,366 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-20T06:39:55,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:55,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:55,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741827_1003 (size=196) 2024-11-20T06:39:55,903 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:39:55,905 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T06:39:55,924 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:39:55,929 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:39:55,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741828_1004 (size=1189) 2024-11-20T06:39:56,384 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store 2024-11-20T06:39:56,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741829_1005 (size=34) 2024-11-20T06:39:56,807 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T06:39:56,810 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:39:56,812 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T06:39:56,812 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:39:56,812 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:39:56,814 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T06:39:56,814 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:39:56,814 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:39:56,816 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732084796812Disabling compacts and flushes for region at 1732084796812Disabling writes for close at 1732084796814 (+2 ms)Writing region close event to WAL at 1732084796814Closed at 1732084796814 2024-11-20T06:39:56,819 WARN [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/.initializing 2024-11-20T06:39:56,819 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/WALs/61c2066a8a0f,33659,1732084793752 2024-11-20T06:39:56,828 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T06:39:56,845 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C33659%2C1732084793752, suffix=, logDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/WALs/61c2066a8a0f,33659,1732084793752, archiveDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/oldWALs, maxLogs=10 2024-11-20T06:39:56,872 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/WALs/61c2066a8a0f,33659,1732084793752/61c2066a8a0f%2C33659%2C1732084793752.1732084796850, exclude list is [], retry=0 2024-11-20T06:39:56,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-11a6f10b-a11f-44f5-a877-72f92b360c7e,DISK] 2024-11-20T06:39:56,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T06:39:56,932 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/WALs/61c2066a8a0f,33659,1732084793752/61c2066a8a0f%2C33659%2C1732084793752.1732084796850 2024-11-20T06:39:56,934 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38513:38513)] 2024-11-20T06:39:56,934 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:39:56,935 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:39:56,939 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:56,940 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:56,982 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:57,008 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T06:39:57,012 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:57,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:39:57,016 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:57,020 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T06:39:57,021 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:57,022 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:39:57,022 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:57,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T06:39:57,026 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:57,028 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:39:57,028 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:57,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T06:39:57,031 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:57,033 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:39:57,033 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:57,038 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:57,040 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:57,047 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:57,048 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:57,052 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T06:39:57,058 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:39:57,067 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:39:57,070 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62823251, jitterRate=-0.06386061012744904}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T06:39:57,079 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732084796953Initializing all the Stores at 1732084796956 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732084796957 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084796958 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084796958Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084796958Cleaning up temporary data from old regions at 1732084797048 (+90 ms)Region opened successfully at 1732084797078 (+30 ms) 2024-11-20T06:39:57,080 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T06:39:57,120 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4959f1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0 2024-11-20T06:39:57,156 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T06:39:57,171 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T06:39:57,171 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T06:39:57,175 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T06:39:57,178 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-20T06:39:57,184 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T06:39:57,184 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T06:39:57,220 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T06:39:57,231 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T06:39:57,285 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T06:39:57,288 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T06:39:57,291 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T06:39:57,302 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T06:39:57,305 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T06:39:57,311 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T06:39:57,318 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T06:39:57,321 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T06:39:57,327 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T06:39:57,346 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T06:39:57,352 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T06:39:57,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T06:39:57,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T06:39:57,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:57,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:57,365 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c2066a8a0f,33659,1732084793752, sessionid=0x101584211260000, setting cluster-up flag (Was=false) 2024-11-20T06:39:57,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:57,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:57,410 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T06:39:57,413 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,33659,1732084793752 2024-11-20T06:39:57,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:57,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:57,452 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T06:39:57,454 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,33659,1732084793752 2024-11-20T06:39:57,461 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T06:39:57,538 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T06:39:57,543 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(746): ClusterId : f056ae1d-0bc3-461f-a70b-1784272b5485 2024-11-20T06:39:57,545 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T06:39:57,547 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T06:39:57,555 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T06:39:57,561 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c2066a8a0f,33659,1732084793752 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T06:39:57,579 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T06:39:57,579 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T06:39:57,579 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:39:57,580 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:39:57,580 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:39:57,580 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:39:57,580 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c2066a8a0f:0, corePoolSize=10, maxPoolSize=10 2024-11-20T06:39:57,581 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,581 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2 2024-11-20T06:39:57,581 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,582 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732084827582 2024-11-20T06:39:57,584 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T06:39:57,585 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T06:39:57,587 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T06:39:57,587 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T06:39:57,587 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T06:39:57,587 DEBUG [RS:0;61c2066a8a0f:33847 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c90b009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0 2024-11-20T06:39:57,589 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T06:39:57,590 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T06:39:57,590 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T06:39:57,590 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T06:39:57,592 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:57,593 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T06:39:57,592 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,597 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T06:39:57,598 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T06:39:57,599 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T06:39:57,603 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T06:39:57,603 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T06:39:57,603 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c2066a8a0f:33847 2024-11-20T06:39:57,605 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732084797604,5,FailOnTimeoutGroup] 2024-11-20T06:39:57,606 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732084797605,5,FailOnTimeoutGroup] 2024-11-20T06:39:57,606 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,606 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T06:39:57,607 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,607 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T06:39:57,608 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T06:39:57,608 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T06:39:57,608 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741831_1007 (size=1321) 2024-11-20T06:39:57,611 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,33659,1732084793752 with port=33847, startcode=1732084794617 2024-11-20T06:39:57,621 DEBUG [RS:0;61c2066a8a0f:33847 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T06:39:57,650 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60949, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T06:39:57,655 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33659 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,33847,1732084794617 2024-11-20T06:39:57,657 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33659 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,33847,1732084794617 2024-11-20T06:39:57,670 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5 2024-11-20T06:39:57,671 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36267 2024-11-20T06:39:57,671 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T06:39:57,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T06:39:57,703 DEBUG [RS:0;61c2066a8a0f:33847 {}] zookeeper.ZKUtil(111): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,33847,1732084794617 2024-11-20T06:39:57,703 WARN [RS:0;61c2066a8a0f:33847 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T06:39:57,703 INFO [RS:0;61c2066a8a0f:33847 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:39:57,704 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/WALs/61c2066a8a0f,33847,1732084794617 2024-11-20T06:39:57,706 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,33847,1732084794617] 2024-11-20T06:39:57,733 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T06:39:57,748 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T06:39:57,753 INFO [RS:0;61c2066a8a0f:33847 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T06:39:57,753 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,754 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T06:39:57,759 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T06:39:57,760 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,760 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,760 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,761 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,761 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,761 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,761 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2 2024-11-20T06:39:57,761 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,761 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,761 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,762 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,762 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,762 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:39:57,762 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:39:57,762 DEBUG [RS:0;61c2066a8a0f:33847 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:39:57,763 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,763 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,763 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,764 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,764 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,764 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,33847,1732084794617-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T06:39:57,782 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T06:39:57,784 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,33847,1732084794617-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,784 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,785 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.Replication(171): 61c2066a8a0f,33847,1732084794617 started 2024-11-20T06:39:57,802 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:57,802 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,33847,1732084794617, RpcServer on 61c2066a8a0f/172.17.0.2:33847, sessionid=0x101584211260001 2024-11-20T06:39:57,803 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T06:39:57,803 DEBUG [RS:0;61c2066a8a0f:33847 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,33847,1732084794617 2024-11-20T06:39:57,803 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,33847,1732084794617' 2024-11-20T06:39:57,803 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T06:39:57,804 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T06:39:57,805 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T06:39:57,805 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T06:39:57,805 DEBUG [RS:0;61c2066a8a0f:33847 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,33847,1732084794617 2024-11-20T06:39:57,805 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,33847,1732084794617' 2024-11-20T06:39:57,805 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T06:39:57,806 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T06:39:57,807 DEBUG [RS:0;61c2066a8a0f:33847 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T06:39:57,807 INFO [RS:0;61c2066a8a0f:33847 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T06:39:57,807 INFO [RS:0;61c2066a8a0f:33847 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T06:39:57,912 INFO [RS:0;61c2066a8a0f:33847 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T06:39:57,916 INFO [RS:0;61c2066a8a0f:33847 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C33847%2C1732084794617, suffix=, logDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/WALs/61c2066a8a0f,33847,1732084794617, archiveDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/oldWALs, maxLogs=32 2024-11-20T06:39:57,931 DEBUG [RS:0;61c2066a8a0f:33847 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/WALs/61c2066a8a0f,33847,1732084794617/61c2066a8a0f%2C33847%2C1732084794617.1732084797919, exclude list is [], retry=0 2024-11-20T06:39:57,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-11a6f10b-a11f-44f5-a877-72f92b360c7e,DISK] 2024-11-20T06:39:57,939 INFO [RS:0;61c2066a8a0f:33847 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/WALs/61c2066a8a0f,33847,1732084794617/61c2066a8a0f%2C33847%2C1732084794617.1732084797919 2024-11-20T06:39:57,939 DEBUG [RS:0;61c2066a8a0f:33847 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38513:38513)] 2024-11-20T06:39:58,012 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T06:39:58,013 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5 2024-11-20T06:39:58,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741833_1009 (size=32) 2024-11-20T06:39:58,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:39:58,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T06:39:58,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T06:39:58,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:58,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:39:58,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T06:39:58,447 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T06:39:58,447 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:58,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:39:58,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T06:39:58,451 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T06:39:58,451 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:58,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:39:58,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T06:39:58,455 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T06:39:58,456 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:58,456 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:39:58,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T06:39:58,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740 2024-11-20T06:39:58,459 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740 2024-11-20T06:39:58,462 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T06:39:58,462 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T06:39:58,463 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:39:58,465 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T06:39:58,469 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:39:58,470 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73012070, jitterRate=0.08796462416648865}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:39:58,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732084798434Initializing all the Stores at 1732084798439 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732084798439Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732084798439Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084798439Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732084798439Cleaning up temporary data from old regions at 1732084798462 (+23 ms)Region opened successfully at 1732084798472 (+10 ms) 2024-11-20T06:39:58,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T06:39:58,473 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T06:39:58,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T06:39:58,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T06:39:58,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T06:39:58,474 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T06:39:58,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732084798473Disabling compacts and flushes for region at 1732084798473Disabling writes for close at 1732084798473Writing region close event to WAL at 1732084798474 (+1 ms)Closed at 1732084798474 2024-11-20T06:39:58,478 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T06:39:58,478 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T06:39:58,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T06:39:58,491 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T06:39:58,493 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T06:39:58,648 DEBUG [61c2066a8a0f:33659 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T06:39:58,661 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:39:58,666 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,33847,1732084794617, state=OPENING 2024-11-20T06:39:58,718 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T06:39:58,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:58,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:39:58,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:39:58,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:39:58,731 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T06:39:58,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:39:58,914 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T06:39:58,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54761, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T06:39:58,928 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T06:39:58,928 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:39:58,929 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T06:39:58,932 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C33847%2C1732084794617.meta, suffix=.meta, logDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/WALs/61c2066a8a0f,33847,1732084794617, archiveDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/oldWALs, maxLogs=32 2024-11-20T06:39:58,947 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/WALs/61c2066a8a0f,33847,1732084794617/61c2066a8a0f%2C33847%2C1732084794617.meta.1732084798934.meta, exclude list is [], retry=0 2024-11-20T06:39:58,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40437,DS-11a6f10b-a11f-44f5-a877-72f92b360c7e,DISK] 2024-11-20T06:39:58,954 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/WALs/61c2066a8a0f,33847,1732084794617/61c2066a8a0f%2C33847%2C1732084794617.meta.1732084798934.meta 2024-11-20T06:39:58,955 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38513:38513)] 2024-11-20T06:39:58,955 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:39:58,957 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T06:39:58,959 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T06:39:58,963 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T06:39:58,967 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T06:39:58,968 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:39:58,968 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T06:39:58,968 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T06:39:58,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T06:39:58,973 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T06:39:58,973 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:58,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:39:58,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T06:39:58,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T06:39:58,975 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:58,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:39:58,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T06:39:58,978 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T06:39:58,978 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:58,979 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:39:58,979 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T06:39:58,980 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T06:39:58,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:58,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:39:58,982 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T06:39:58,983 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740 2024-11-20T06:39:58,986 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740 2024-11-20T06:39:58,988 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T06:39:58,989 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T06:39:58,990 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:39:58,992 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T06:39:58,994 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70810856, jitterRate=0.05516397953033447}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:39:58,994 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T06:39:58,996 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732084798968Writing region info on filesystem at 1732084798969 (+1 ms)Initializing all the Stores at 1732084798971 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732084798971Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732084798971Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084798971Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732084798971Cleaning up temporary data from old regions at 1732084798989 (+18 ms)Running coprocessor post-open hooks at 1732084798994 (+5 ms)Region opened successfully at 1732084798996 (+2 ms) 2024-11-20T06:39:59,002 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732084798906 2024-11-20T06:39:59,013 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T06:39:59,014 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T06:39:59,016 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:39:59,019 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,33847,1732084794617, state=OPEN 2024-11-20T06:39:59,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:39:59,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:39:59,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:39:59,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:39:59,074 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:39:59,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T06:39:59,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,33847,1732084794617 in 339 msec 2024-11-20T06:39:59,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T06:39:59,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 598 msec 2024-11-20T06:39:59,089 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T06:39:59,089 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T06:39:59,108 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:39:59,109 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:39:59,128 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:39:59,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37835, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:39:59,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6500 sec 2024-11-20T06:39:59,151 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732084799151, completionTime=-1 2024-11-20T06:39:59,154 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T06:39:59,154 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T06:39:59,176 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-20T06:39:59,177 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732084859177 2024-11-20T06:39:59,177 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732084919177 2024-11-20T06:39:59,177 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-20T06:39:59,179 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,33659,1732084793752-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:59,180 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,33659,1732084793752-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:59,180 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,33659,1732084793752-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:59,181 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c2066a8a0f:33659, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:59,182 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:59,182 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:59,188 DEBUG [master/61c2066a8a0f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T06:39:59,209 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.405sec 2024-11-20T06:39:59,210 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T06:39:59,211 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T06:39:59,212 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T06:39:59,213 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T06:39:59,213 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T06:39:59,213 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,33659,1732084793752-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T06:39:59,214 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,33659,1732084793752-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T06:39:59,222 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T06:39:59,223 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T06:39:59,224 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,33659,1732084793752-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:39:59,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e129cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:39:59,255 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T06:39:59,255 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T06:39:59,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:39:59,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:39:59,267 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:39:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:39:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:39:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c924b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:39:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:39:59,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:39:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:39:59,277 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:39:59,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebea588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:39:59,280 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:39:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:39:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:39:59,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:39:59,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c2066a8a0f,33659,1732084793752 2024-11-20T06:39:59,319 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=323, ProcessCount=11, AvailableMemoryMB=5165 2024-11-20T06:39:59,328 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T06:39:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,33659,1732084793752 2024-11-20T06:39:59,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-11-20T06:39:59,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T06:39:59,356 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T06:39:59,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T06:39:59,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:39:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T06:39:59,379 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T06:39:59,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-20T06:39:59,385 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:39:59,388 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T06:39:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:39:59,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741835_1011 (size=959) 2024-11-20T06:39:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:39:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:39:59,806 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5 2024-11-20T06:39:59,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741836_1012 (size=53) 2024-11-20T06:40:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:40:00,218 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:40:00,218 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8b50cf3b5c22a8ba480f30fa5ecd19ec, disabling compactions & flushes 2024-11-20T06:40:00,218 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:00,218 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:00,219 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. after waiting 0 ms 2024-11-20T06:40:00,219 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:00,219 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:00,219 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: Waiting for close lock at 1732084800218Disabling compacts and flushes for region at 1732084800218Disabling writes for close at 1732084800219 (+1 ms)Writing region close event to WAL at 1732084800219Closed at 1732084800219 2024-11-20T06:40:00,224 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T06:40:00,231 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732084800224"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732084800224"}]},"ts":"1732084800224"} 2024-11-20T06:40:00,236 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T06:40:00,238 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T06:40:00,240 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084800238"}]},"ts":"1732084800238"} 2024-11-20T06:40:00,244 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T06:40:00,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b50cf3b5c22a8ba480f30fa5ecd19ec, ASSIGN}] 2024-11-20T06:40:00,250 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b50cf3b5c22a8ba480f30fa5ecd19ec, ASSIGN 2024-11-20T06:40:00,252 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b50cf3b5c22a8ba480f30fa5ecd19ec, ASSIGN; state=OFFLINE, location=61c2066a8a0f,33847,1732084794617; forceNewPlan=false, retain=false 2024-11-20T06:40:00,405 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8b50cf3b5c22a8ba480f30fa5ecd19ec, regionState=OPENING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:00,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b50cf3b5c22a8ba480f30fa5ecd19ec, ASSIGN because future has completed 2024-11-20T06:40:00,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:40:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:40:00,579 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:00,579 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:40:00,580 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,580 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:40:00,580 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,580 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,583 INFO [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,585 INFO [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b50cf3b5c22a8ba480f30fa5ecd19ec columnFamilyName A 2024-11-20T06:40:00,585 DEBUG [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:00,586 INFO [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] regionserver.HStore(327): Store=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:00,586 INFO [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,589 INFO [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b50cf3b5c22a8ba480f30fa5ecd19ec columnFamilyName B 2024-11-20T06:40:00,589 DEBUG [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:00,589 INFO [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] regionserver.HStore(327): Store=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:00,590 INFO [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,592 INFO [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b50cf3b5c22a8ba480f30fa5ecd19ec columnFamilyName C 2024-11-20T06:40:00,592 DEBUG [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:00,593 INFO [StoreOpener-8b50cf3b5c22a8ba480f30fa5ecd19ec-1 {}] regionserver.HStore(327): Store=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:00,593 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,595 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,596 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,598 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,598 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,599 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:40:00,602 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,606 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:40:00,607 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8b50cf3b5c22a8ba480f30fa5ecd19ec; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71698494, jitterRate=0.06839081645011902}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:40:00,607 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:00,609 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: Running coprocessor pre-open hook at 1732084800580Writing region info on filesystem at 1732084800580Initializing all the Stores at 1732084800582 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084800582Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084800582Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084800582Cleaning up temporary data from old regions at 1732084800598 (+16 ms)Running coprocessor post-open hooks at 1732084800607 (+9 ms)Region opened successfully at 1732084800609 (+2 ms) 2024-11-20T06:40:00,611 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., pid=6, masterSystemTime=1732084800570 2024-11-20T06:40:00,614 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:00,614 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:00,615 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8b50cf3b5c22a8ba480f30fa5ecd19ec, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:00,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:40:00,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T06:40:00,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 in 206 msec 2024-11-20T06:40:00,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T06:40:00,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b50cf3b5c22a8ba480f30fa5ecd19ec, ASSIGN in 380 msec 2024-11-20T06:40:00,631 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T06:40:00,632 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084800632"}]},"ts":"1732084800632"} 2024-11-20T06:40:00,635 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T06:40:00,637 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T06:40:00,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2700 sec 2024-11-20T06:40:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:40:01,545 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:01,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b3430e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:01,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:01,552 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfcdc6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:01,556 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0d8197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,558 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:01,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:01,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:01,565 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T06:40:01,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,33659,1732084793752 2024-11-20T06:40:01,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@69abe2c9 2024-11-20T06:40:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T06:40:01,571 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T06:40:01,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:01,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:01,578 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@455d50bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:01,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760c0989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:01,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:01,586 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3d2fd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:01,589 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:01,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c860641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:01,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:01,593 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:01,597 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:01,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:01,601 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b084ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:01,604 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:01,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2815dd8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f5332d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:01,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:01,607 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70343832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:01,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:01,609 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:01,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:01,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:01,613 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:01,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:01,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:01,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6891f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:01,616 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe9eb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19215e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,618 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:01,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:01,620 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:01,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291ab3a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:01,622 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d11e8ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:01,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:01,626 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d774ab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:01,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:01,629 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94111c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:01,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:01,632 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355546b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:01,635 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23600c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:01,640 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:01,640 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:01,640 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:01,641 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:01,641 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:01,642 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:01,643 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:01,643 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:01,643 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:01,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:01,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:01,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:01,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:01,646 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:01,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:01,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:01,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:01,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:01,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:01,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:01,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:01,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-20T06:40:01,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:01,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:01,659 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T06:40:01,663 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:01,666 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:01,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:01,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:01,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:01,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:01,743 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:01,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T06:40:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T06:40:01,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T06:40:01,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:01,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:01,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:01,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:01,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/259c19a3cea64e83871dfed22ee3dbf3 is 50, key is test_row_0/A:col10/1732084801741/Put/seqid=0 2024-11-20T06:40:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:01,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084861864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:01,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084861865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:01,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084861875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:01,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084861880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:01,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084861882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:01,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741837_1013 (size=11997) 2024-11-20T06:40:01,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/259c19a3cea64e83871dfed22ee3dbf3 2024-11-20T06:40:01,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T06:40:02,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T06:40:02,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084862036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084862037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:02,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084862037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084862038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084862038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/93f8480ec219402e8e52ae569a5800e4 is 50, key is test_row_0/B:col10/1732084801741/Put/seqid=0 2024-11-20T06:40:02,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741838_1014 (size=9657) 2024-11-20T06:40:02,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/93f8480ec219402e8e52ae569a5800e4 2024-11-20T06:40:02,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/03e230f7bfec40319d10277e8ce34f57 is 50, key is test_row_0/C:col10/1732084801741/Put/seqid=0 2024-11-20T06:40:02,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T06:40:02,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741839_1015 (size=9657) 2024-11-20T06:40:02,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:02,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084862255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084862257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084862258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084862260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084862260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T06:40:02,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T06:40:02,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:02,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T06:40:02,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084862578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084862579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084862580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084862581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084862580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:02,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/03e230f7bfec40319d10277e8ce34f57 2024-11-20T06:40:02,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/259c19a3cea64e83871dfed22ee3dbf3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/259c19a3cea64e83871dfed22ee3dbf3 2024-11-20T06:40:02,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/259c19a3cea64e83871dfed22ee3dbf3, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T06:40:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/93f8480ec219402e8e52ae569a5800e4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/93f8480ec219402e8e52ae569a5800e4 2024-11-20T06:40:02,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T06:40:02,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:02,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:02,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/93f8480ec219402e8e52ae569a5800e4, entries=100, sequenceid=15, filesize=9.4 K 2024-11-20T06:40:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/03e230f7bfec40319d10277e8ce34f57 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/03e230f7bfec40319d10277e8ce34f57 2024-11-20T06:40:02,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/03e230f7bfec40319d10277e8ce34f57, entries=100, sequenceid=15, filesize=9.4 K 2024-11-20T06:40:02,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 979ms, sequenceid=15, compaction requested=false 2024-11-20T06:40:02,731 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T06:40:02,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T06:40:02,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T06:40:02,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:02,843 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T06:40:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/3a06d0e3deac413ba3303a75122eac0f is 50, key is test_row_0/A:col10/1732084801876/Put/seqid=0 2024-11-20T06:40:02,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741840_1016 (size=12001) 2024-11-20T06:40:02,876 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/3a06d0e3deac413ba3303a75122eac0f 2024-11-20T06:40:02,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/89e319f70a354bb3b4a6d719f45d6f5f is 50, key is test_row_0/B:col10/1732084801876/Put/seqid=0 2024-11-20T06:40:02,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741841_1017 (size=12001) 2024-11-20T06:40:02,934 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/89e319f70a354bb3b4a6d719f45d6f5f 2024-11-20T06:40:02,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/fb28761e24424eb69352109b70d76ed2 is 50, key is test_row_0/C:col10/1732084801876/Put/seqid=0 2024-11-20T06:40:02,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741842_1018 (size=12001) 2024-11-20T06:40:02,989 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/fb28761e24424eb69352109b70d76ed2 2024-11-20T06:40:03,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/3a06d0e3deac413ba3303a75122eac0f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/3a06d0e3deac413ba3303a75122eac0f 2024-11-20T06:40:03,019 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/3a06d0e3deac413ba3303a75122eac0f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T06:40:03,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/89e319f70a354bb3b4a6d719f45d6f5f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/89e319f70a354bb3b4a6d719f45d6f5f 2024-11-20T06:40:03,048 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/89e319f70a354bb3b4a6d719f45d6f5f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T06:40:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/fb28761e24424eb69352109b70d76ed2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/fb28761e24424eb69352109b70d76ed2 2024-11-20T06:40:03,062 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/fb28761e24424eb69352109b70d76ed2, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T06:40:03,065 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 223ms, sequenceid=38, compaction requested=false 2024-11-20T06:40:03,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:03,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:03,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-20T06:40:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-20T06:40:03,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T06:40:03,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-20T06:40:03,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.4310 sec 2024-11-20T06:40:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:03,128 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:03,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:40:03,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/19020a929339461f86aac585aae39eeb is 50, key is test_row_0/A:col10/1732084803105/Put/seqid=0 2024-11-20T06:40:03,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741843_1019 (size=21365) 2024-11-20T06:40:03,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/19020a929339461f86aac585aae39eeb 2024-11-20T06:40:03,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/08eb030ec17247bb84c1fa4bb94e349f is 50, key is test_row_0/B:col10/1732084803105/Put/seqid=0 2024-11-20T06:40:03,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741844_1020 (size=12001) 2024-11-20T06:40:03,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/08eb030ec17247bb84c1fa4bb94e349f 2024-11-20T06:40:03,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084863225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/591886a93388441cb5c9b11cfddc4387 is 50, key is test_row_0/C:col10/1732084803105/Put/seqid=0 2024-11-20T06:40:03,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084863232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084863235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084863239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084863244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741845_1021 (size=12001) 2024-11-20T06:40:03,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/591886a93388441cb5c9b11cfddc4387 2024-11-20T06:40:03,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/19020a929339461f86aac585aae39eeb as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/19020a929339461f86aac585aae39eeb 2024-11-20T06:40:03,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/19020a929339461f86aac585aae39eeb, entries=350, sequenceid=50, filesize=20.9 K 2024-11-20T06:40:03,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/08eb030ec17247bb84c1fa4bb94e349f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/08eb030ec17247bb84c1fa4bb94e349f 2024-11-20T06:40:03,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/08eb030ec17247bb84c1fa4bb94e349f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T06:40:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/591886a93388441cb5c9b11cfddc4387 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/591886a93388441cb5c9b11cfddc4387 2024-11-20T06:40:03,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/591886a93388441cb5c9b11cfddc4387, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T06:40:03,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 235ms, sequenceid=50, compaction requested=true 2024-11-20T06:40:03,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:03,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:03,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:03,373 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:03,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:03,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:03,373 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:03,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:03,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:03,373 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:03,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T06:40:03,377 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:03,379 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:03,379 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:03,380 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/93f8480ec219402e8e52ae569a5800e4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/89e319f70a354bb3b4a6d719f45d6f5f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/08eb030ec17247bb84c1fa4bb94e349f] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=32.9 K 2024-11-20T06:40:03,384 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f8480ec219402e8e52ae569a5800e4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732084801739 2024-11-20T06:40:03,385 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:03,385 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:03,385 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:03,386 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/259c19a3cea64e83871dfed22ee3dbf3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/3a06d0e3deac413ba3303a75122eac0f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/19020a929339461f86aac585aae39eeb] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=44.3 K 2024-11-20T06:40:03,390 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 89e319f70a354bb3b4a6d719f45d6f5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732084801857 2024-11-20T06:40:03,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b6abf57731f6466e81a274c29d143d52 is 50, key is test_row_0/A:col10/1732084803367/Put/seqid=0 2024-11-20T06:40:03,391 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 08eb030ec17247bb84c1fa4bb94e349f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084803105 2024-11-20T06:40:03,392 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 259c19a3cea64e83871dfed22ee3dbf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732084801721 2024-11-20T06:40:03,394 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a06d0e3deac413ba3303a75122eac0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732084801857 2024-11-20T06:40:03,398 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19020a929339461f86aac585aae39eeb, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084803105 2024-11-20T06:40:03,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084863392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084863394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084863396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084863397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084863403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741846_1022 (size=19021) 2024-11-20T06:40:03,456 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:03,458 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/77ba247a45cc4d418118311fdee6a57a is 50, key is test_row_0/B:col10/1732084803105/Put/seqid=0 2024-11-20T06:40:03,461 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:03,462 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/630d9877382a4171a54fae730e15edd2 is 50, key is test_row_0/A:col10/1732084803105/Put/seqid=0 2024-11-20T06:40:03,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741847_1023 (size=12104) 2024-11-20T06:40:03,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741848_1024 (size=12104) 2024-11-20T06:40:03,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,521 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/630d9877382a4171a54fae730e15edd2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/630d9877382a4171a54fae730e15edd2 2024-11-20T06:40:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084863515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084863516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084863516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084863517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084863517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,544 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 630d9877382a4171a54fae730e15edd2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:03,545 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:03,548 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=13, startTime=1732084803369; duration=0sec 2024-11-20T06:40:03,549 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:03,549 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:03,549 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:03,552 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:03,552 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:03,552 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:03,552 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/03e230f7bfec40319d10277e8ce34f57, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/fb28761e24424eb69352109b70d76ed2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/591886a93388441cb5c9b11cfddc4387] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=32.9 K 2024-11-20T06:40:03,553 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e230f7bfec40319d10277e8ce34f57, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732084801739 2024-11-20T06:40:03,553 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb28761e24424eb69352109b70d76ed2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732084801857 2024-11-20T06:40:03,554 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 591886a93388441cb5c9b11cfddc4387, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084803105 2024-11-20T06:40:03,579 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:03,580 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/c649333dfe7f4ba08b42fee158186666 is 50, key is test_row_0/C:col10/1732084803105/Put/seqid=0 2024-11-20T06:40:03,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741849_1025 (size=12104) 2024-11-20T06:40:03,606 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/c649333dfe7f4ba08b42fee158186666 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/c649333dfe7f4ba08b42fee158186666 2024-11-20T06:40:03,623 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into c649333dfe7f4ba08b42fee158186666(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:03,624 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:03,624 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=13, startTime=1732084803373; duration=0sec 2024-11-20T06:40:03,624 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:03,624 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:03,728 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T06:40:03,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084863734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084863745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084863746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084863746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084863747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T06:40:03,805 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:03,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-20T06:40:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T06:40:03,813 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:03,817 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:03,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:03,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b6abf57731f6466e81a274c29d143d52 2024-11-20T06:40:03,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/7a06c40fe8d441ecb1ae6ac53905abf3 is 50, key is test_row_0/B:col10/1732084803367/Put/seqid=0 2024-11-20T06:40:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T06:40:03,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741850_1026 (size=12001) 2024-11-20T06:40:03,919 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/77ba247a45cc4d418118311fdee6a57a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/77ba247a45cc4d418118311fdee6a57a 2024-11-20T06:40:03,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/7a06c40fe8d441ecb1ae6ac53905abf3 2024-11-20T06:40:03,936 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 77ba247a45cc4d418118311fdee6a57a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:03,936 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:03,936 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=13, startTime=1732084803373; duration=0sec 2024-11-20T06:40:03,936 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:03,937 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:03,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/83e0afc73945441e9c077e4f4c2d471f is 50, key is test_row_0/C:col10/1732084803367/Put/seqid=0 2024-11-20T06:40:03,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T06:40:03,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:03,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:03,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:03,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:03,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:04,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741851_1027 (size=12001) 2024-11-20T06:40:04,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/83e0afc73945441e9c077e4f4c2d471f 2024-11-20T06:40:04,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b6abf57731f6466e81a274c29d143d52 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b6abf57731f6466e81a274c29d143d52 2024-11-20T06:40:04,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b6abf57731f6466e81a274c29d143d52, entries=300, sequenceid=77, filesize=18.6 K 2024-11-20T06:40:04,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/7a06c40fe8d441ecb1ae6ac53905abf3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7a06c40fe8d441ecb1ae6ac53905abf3 2024-11-20T06:40:04,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084864056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084864057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084864057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7a06c40fe8d441ecb1ae6ac53905abf3, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T06:40:04,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084864057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/83e0afc73945441e9c077e4f4c2d471f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/83e0afc73945441e9c077e4f4c2d471f 2024-11-20T06:40:04,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084864065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/83e0afc73945441e9c077e4f4c2d471f, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T06:40:04,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 720ms, sequenceid=77, compaction requested=false 2024-11-20T06:40:04,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T06:40:04,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T06:40:04,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:04,135 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-20T06:40:04,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/1c49477c8e4449c9b257d251ae2b87bc is 50, key is test_row_0/A:col10/1732084803378/Put/seqid=0 2024-11-20T06:40:04,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741852_1028 (size=12001) 2024-11-20T06:40:04,175 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/1c49477c8e4449c9b257d251ae2b87bc 2024-11-20T06:40:04,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/4efef04d3ec64700a36796476662bbfe is 50, key is test_row_0/B:col10/1732084803378/Put/seqid=0 2024-11-20T06:40:04,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741853_1029 (size=12001) 2024-11-20T06:40:04,227 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/4efef04d3ec64700a36796476662bbfe 2024-11-20T06:40:04,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/3cfd2910d45348eb86025c5ce752a3d5 is 50, key is test_row_0/C:col10/1732084803378/Put/seqid=0 2024-11-20T06:40:04,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741854_1030 (size=12001) 2024-11-20T06:40:04,310 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/3cfd2910d45348eb86025c5ce752a3d5 2024-11-20T06:40:04,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/1c49477c8e4449c9b257d251ae2b87bc as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1c49477c8e4449c9b257d251ae2b87bc 2024-11-20T06:40:04,343 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1c49477c8e4449c9b257d251ae2b87bc, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T06:40:04,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/4efef04d3ec64700a36796476662bbfe as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4efef04d3ec64700a36796476662bbfe 2024-11-20T06:40:04,365 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4efef04d3ec64700a36796476662bbfe, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T06:40:04,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/3cfd2910d45348eb86025c5ce752a3d5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/3cfd2910d45348eb86025c5ce752a3d5 2024-11-20T06:40:04,389 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/3cfd2910d45348eb86025c5ce752a3d5, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T06:40:04,392 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 256ms, sequenceid=89, compaction requested=true 2024-11-20T06:40:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-20T06:40:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-20T06:40:04,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-20T06:40:04,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 578 msec 2024-11-20T06:40:04,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 599 msec 2024-11-20T06:40:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T06:40:04,435 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:04,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-20T06:40:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T06:40:04,443 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:04,445 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:04,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:04,484 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T06:40:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T06:40:04,585 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T06:40:04,586 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T06:40:04,589 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T06:40:04,589 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T06:40:04,589 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T06:40:04,590 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T06:40:04,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T06:40:04,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T06:40:04,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T06:40:04,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:04,605 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T06:40:04,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:04,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/d54918dce2724f87af256449b1ebd226 is 50, key is test_row_0/A:col10/1732084804598/Put/seqid=0 2024-11-20T06:40:04,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741855_1031 (size=12001) 2024-11-20T06:40:04,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084864678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084864678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084864680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084864689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084864686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T06:40:04,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084864794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084864794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084864804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084864804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:04,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084864814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:04,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084865004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084865014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084865014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084865014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084865039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T06:40:05,085 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/d54918dce2724f87af256449b1ebd226 2024-11-20T06:40:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/e5c93e7da6d34b04b541167cfc9bf21a is 50, key is test_row_0/B:col10/1732084804598/Put/seqid=0 2024-11-20T06:40:05,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741856_1032 (size=12001) 2024-11-20T06:40:05,196 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/e5c93e7da6d34b04b541167cfc9bf21a 2024-11-20T06:40:05,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/db4f78a822f44c8cac1d4084a0f8aef0 is 50, key is test_row_0/C:col10/1732084804598/Put/seqid=0 2024-11-20T06:40:05,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741857_1033 (size=12001) 2024-11-20T06:40:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084865314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084865344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084865348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084865346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084865355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T06:40:05,686 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/db4f78a822f44c8cac1d4084a0f8aef0 2024-11-20T06:40:05,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/d54918dce2724f87af256449b1ebd226 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/d54918dce2724f87af256449b1ebd226 2024-11-20T06:40:05,716 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/d54918dce2724f87af256449b1ebd226, entries=150, sequenceid=99, filesize=11.7 K 2024-11-20T06:40:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/e5c93e7da6d34b04b541167cfc9bf21a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e5c93e7da6d34b04b541167cfc9bf21a 2024-11-20T06:40:05,732 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e5c93e7da6d34b04b541167cfc9bf21a, entries=150, sequenceid=99, filesize=11.7 K 2024-11-20T06:40:05,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/db4f78a822f44c8cac1d4084a0f8aef0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/db4f78a822f44c8cac1d4084a0f8aef0 2024-11-20T06:40:05,800 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/db4f78a822f44c8cac1d4084a0f8aef0, entries=150, sequenceid=99, filesize=11.7 K 2024-11-20T06:40:05,808 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 1203ms, sequenceid=99, compaction requested=true 2024-11-20T06:40:05,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:05,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:05,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-20T06:40:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-20T06:40:05,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T06:40:05,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3710 sec 2024-11-20T06:40:05,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3890 sec 2024-11-20T06:40:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:05,835 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:05,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-20T06:40:05,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/05215c616f4f4e8bb9a5da0f2517c88e is 50, key is test_row_0/A:col10/1732084804685/Put/seqid=0 2024-11-20T06:40:05,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084865865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084865870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084865874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084865874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084865878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:05,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741858_1034 (size=14341) 2024-11-20T06:40:05,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084865985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084866015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:06,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084866016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:06,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:06,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084866194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:06,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084866224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:06,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084866224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:06,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/05215c616f4f4e8bb9a5da0f2517c88e 2024-11-20T06:40:06,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/c3be69b1548443aa92d700a0e39df1a2 is 50, key is test_row_0/B:col10/1732084804685/Put/seqid=0 2024-11-20T06:40:06,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741859_1035 (size=12001) 2024-11-20T06:40:06,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/c3be69b1548443aa92d700a0e39df1a2 2024-11-20T06:40:06,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/377415db16c949b4b1344a151c63b88c is 50, key is test_row_0/C:col10/1732084804685/Put/seqid=0 2024-11-20T06:40:06,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741860_1036 (size=12001) 2024-11-20T06:40:06,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084866514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:06,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/377415db16c949b4b1344a151c63b88c 2024-11-20T06:40:06,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/05215c616f4f4e8bb9a5da0f2517c88e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/05215c616f4f4e8bb9a5da0f2517c88e 2024-11-20T06:40:06,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/05215c616f4f4e8bb9a5da0f2517c88e, entries=200, sequenceid=126, filesize=14.0 K 2024-11-20T06:40:06,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084866544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:06,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:06,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084866545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:06,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/c3be69b1548443aa92d700a0e39df1a2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c3be69b1548443aa92d700a0e39df1a2 2024-11-20T06:40:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:06,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c3be69b1548443aa92d700a0e39df1a2, entries=150, sequenceid=126, filesize=11.7 K 2024-11-20T06:40:06,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/377415db16c949b4b1344a151c63b88c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/377415db16c949b4b1344a151c63b88c 2024-11-20T06:40:06,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/377415db16c949b4b1344a151c63b88c, entries=150, sequenceid=126, filesize=11.7 K 2024-11-20T06:40:06,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 744ms, sequenceid=126, compaction requested=true 2024-11-20T06:40:06,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:06,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:06,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:06,580 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T06:40:06,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:06,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:06,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:06,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T06:40:06,580 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T06:40:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T06:40:06,585 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:06,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:06,587 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60108 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T06:40:06,587 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 69468 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T06:40:06,587 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:06,587 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:06,587 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/77ba247a45cc4d418118311fdee6a57a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7a06c40fe8d441ecb1ae6ac53905abf3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4efef04d3ec64700a36796476662bbfe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e5c93e7da6d34b04b541167cfc9bf21a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c3be69b1548443aa92d700a0e39df1a2] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=58.7 K 2024-11-20T06:40:06,587 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:06,587 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:06,588 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/630d9877382a4171a54fae730e15edd2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b6abf57731f6466e81a274c29d143d52, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1c49477c8e4449c9b257d251ae2b87bc, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/d54918dce2724f87af256449b1ebd226, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/05215c616f4f4e8bb9a5da0f2517c88e] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=67.8 K 2024-11-20T06:40:06,588 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 77ba247a45cc4d418118311fdee6a57a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084803105 2024-11-20T06:40:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-20T06:40:06,589 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 630d9877382a4171a54fae730e15edd2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084803105 2024-11-20T06:40:06,589 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a06c40fe8d441ecb1ae6ac53905abf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732084803361 2024-11-20T06:40:06,589 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6abf57731f6466e81a274c29d143d52, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732084803227 2024-11-20T06:40:06,590 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c49477c8e4449c9b257d251ae2b87bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732084803378 2024-11-20T06:40:06,591 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:06,591 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting d54918dce2724f87af256449b1ebd226, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732084804581 2024-11-20T06:40:06,592 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05215c616f4f4e8bb9a5da0f2517c88e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732084804673 2024-11-20T06:40:06,592 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 4efef04d3ec64700a36796476662bbfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732084803378 2024-11-20T06:40:06,593 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:06,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T06:40:06,596 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting e5c93e7da6d34b04b541167cfc9bf21a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732084804581 2024-11-20T06:40:06,598 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c3be69b1548443aa92d700a0e39df1a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732084804674 2024-11-20T06:40:06,642 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#24 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:06,643 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/df3d54bc08bf48e88c4b88bb5e0c5bd7 is 50, key is test_row_0/B:col10/1732084804685/Put/seqid=0 2024-11-20T06:40:06,654 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:06,655 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/5d5b6be1d75b49189840b18d26249838 is 50, key is test_row_0/A:col10/1732084804685/Put/seqid=0 2024-11-20T06:40:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T06:40:06,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741861_1037 (size=12275) 2024-11-20T06:40:06,720 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/df3d54bc08bf48e88c4b88bb5e0c5bd7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/df3d54bc08bf48e88c4b88bb5e0c5bd7 2024-11-20T06:40:06,734 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into df3d54bc08bf48e88c4b88bb5e0c5bd7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:06,734 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:06,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741862_1038 (size=12275) 2024-11-20T06:40:06,734 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=11, startTime=1732084806580; duration=0sec 2024-11-20T06:40:06,734 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:06,734 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:06,734 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T06:40:06,740 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60108 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T06:40:06,740 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:06,740 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:06,740 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/c649333dfe7f4ba08b42fee158186666, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/83e0afc73945441e9c077e4f4c2d471f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/3cfd2910d45348eb86025c5ce752a3d5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/db4f78a822f44c8cac1d4084a0f8aef0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/377415db16c949b4b1344a151c63b88c] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=58.7 K 2024-11-20T06:40:06,742 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c649333dfe7f4ba08b42fee158186666, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084803105 2024-11-20T06:40:06,743 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e0afc73945441e9c077e4f4c2d471f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732084803361 2024-11-20T06:40:06,744 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cfd2910d45348eb86025c5ce752a3d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732084803378 2024-11-20T06:40:06,745 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting db4f78a822f44c8cac1d4084a0f8aef0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732084804581 2024-11-20T06:40:06,745 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 377415db16c949b4b1344a151c63b88c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732084804674 2024-11-20T06:40:06,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T06:40:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:06,753 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-20T06:40:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/e2e4a61d7c544c7f87b8e6d4cea1628c is 50, key is test_row_0/A:col10/1732084805874/Put/seqid=0 2024-11-20T06:40:06,782 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:06,783 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/dd898e992e8746ec9ffacb020e4ab0fb is 50, key is test_row_0/C:col10/1732084804685/Put/seqid=0 2024-11-20T06:40:06,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741864_1040 (size=12275) 2024-11-20T06:40:06,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741863_1039 (size=12151) 2024-11-20T06:40:06,848 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/e2e4a61d7c544c7f87b8e6d4cea1628c 2024-11-20T06:40:06,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/e242f1b8a712494e8437d5791e6fda21 is 50, key is test_row_0/B:col10/1732084805874/Put/seqid=0 2024-11-20T06:40:06,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:06,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741865_1041 (size=12151) 2024-11-20T06:40:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T06:40:06,916 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/e242f1b8a712494e8437d5791e6fda21 2024-11-20T06:40:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/7255d4ecd50d4bbf9893dfb33a437815 is 50, key is test_row_0/C:col10/1732084805874/Put/seqid=0 2024-11-20T06:40:06,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741866_1042 (size=12151) 2024-11-20T06:40:06,981 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/7255d4ecd50d4bbf9893dfb33a437815 2024-11-20T06:40:06,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/e2e4a61d7c544c7f87b8e6d4cea1628c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e2e4a61d7c544c7f87b8e6d4cea1628c 2024-11-20T06:40:07,008 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e2e4a61d7c544c7f87b8e6d4cea1628c, entries=150, sequenceid=136, filesize=11.9 K 2024-11-20T06:40:07,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/e242f1b8a712494e8437d5791e6fda21 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e242f1b8a712494e8437d5791e6fda21 2024-11-20T06:40:07,029 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e242f1b8a712494e8437d5791e6fda21, entries=150, sequenceid=136, filesize=11.9 K 2024-11-20T06:40:07,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/7255d4ecd50d4bbf9893dfb33a437815 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7255d4ecd50d4bbf9893dfb33a437815 2024-11-20T06:40:07,069 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7255d4ecd50d4bbf9893dfb33a437815, entries=150, sequenceid=136, filesize=11.9 K 2024-11-20T06:40:07,073 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=147.60 KB/151140 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 320ms, sequenceid=136, compaction requested=false 2024-11-20T06:40:07,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:07,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:07,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-20T06:40:07,074 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:07,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T06:40:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-20T06:40:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:07,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T06:40:07,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 483 msec 2024-11-20T06:40:07,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 499 msec 2024-11-20T06:40:07,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/358c9ffaea384cc7866a2f277c8b9faa is 50, key is test_row_0/A:col10/1732084807069/Put/seqid=0 2024-11-20T06:40:07,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741867_1043 (size=16931) 2024-11-20T06:40:07,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084867100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084867102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,146 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/5d5b6be1d75b49189840b18d26249838 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/5d5b6be1d75b49189840b18d26249838 2024-11-20T06:40:07,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084867133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084867127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,159 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 5d5b6be1d75b49189840b18d26249838(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:07,159 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:07,159 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=11, startTime=1732084806580; duration=0sec 2024-11-20T06:40:07,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084867144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,159 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:07,159 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T06:40:07,225 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:07,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-20T06:40:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T06:40:07,229 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:07,230 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:07,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084867244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,261 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/dd898e992e8746ec9ffacb020e4ab0fb as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/dd898e992e8746ec9ffacb020e4ab0fb 2024-11-20T06:40:07,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084867254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084867264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084867265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084867266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,288 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into dd898e992e8746ec9ffacb020e4ab0fb(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:07,288 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:07,288 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=11, startTime=1732084806580; duration=0sec 2024-11-20T06:40:07,288 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:07,288 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T06:40:07,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T06:40:07,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:07,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:07,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:07,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084867464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084867474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084867475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084867475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084867494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/358c9ffaea384cc7866a2f277c8b9faa 2024-11-20T06:40:07,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T06:40:07,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:07,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:07,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:07,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T06:40:07,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/a09dc4b6ed3343e2a7ef1506c0af7fd3 is 50, key is test_row_0/B:col10/1732084807069/Put/seqid=0 2024-11-20T06:40:07,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741868_1044 (size=12151) 2024-11-20T06:40:07,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T06:40:07,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:07,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:07,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:07,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084867774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084867786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084867786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084867788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:07,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084867804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:07,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T06:40:07,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:07,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:07,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:07,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T06:40:07,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/a09dc4b6ed3343e2a7ef1506c0af7fd3 2024-11-20T06:40:08,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/8b53c76cd62040a88820bc10d350df2f is 50, key is test_row_0/C:col10/1732084807069/Put/seqid=0 2024-11-20T06:40:08,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T06:40:08,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:08,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741869_1045 (size=12151) 2024-11-20T06:40:08,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T06:40:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084868284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:08,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:08,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:08,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084868294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:08,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084868294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:08,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:08,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084868304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:08,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:08,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T06:40:08,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:08,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084868325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T06:40:08,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/8b53c76cd62040a88820bc10d350df2f 2024-11-20T06:40:08,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T06:40:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:08,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/358c9ffaea384cc7866a2f277c8b9faa as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/358c9ffaea384cc7866a2f277c8b9faa 2024-11-20T06:40:08,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/358c9ffaea384cc7866a2f277c8b9faa, entries=250, sequenceid=162, filesize=16.5 K 2024-11-20T06:40:08,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/a09dc4b6ed3343e2a7ef1506c0af7fd3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/a09dc4b6ed3343e2a7ef1506c0af7fd3 2024-11-20T06:40:08,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/a09dc4b6ed3343e2a7ef1506c0af7fd3, entries=150, sequenceid=162, filesize=11.9 K 2024-11-20T06:40:08,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/8b53c76cd62040a88820bc10d350df2f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b53c76cd62040a88820bc10d350df2f 2024-11-20T06:40:08,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b53c76cd62040a88820bc10d350df2f, entries=150, sequenceid=162, filesize=11.9 K 2024-11-20T06:40:08,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 1452ms, sequenceid=162, compaction requested=true 2024-11-20T06:40:08,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:08,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:08,527 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:08,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:08,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:08,527 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:08,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:08,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:08,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:08,529 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41357 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:08,529 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:08,529 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,529 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:08,529 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/5d5b6be1d75b49189840b18d26249838, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e2e4a61d7c544c7f87b8e6d4cea1628c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/358c9ffaea384cc7866a2f277c8b9faa] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=40.4 K 2024-11-20T06:40:08,529 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:08,529 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,529 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/df3d54bc08bf48e88c4b88bb5e0c5bd7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e242f1b8a712494e8437d5791e6fda21, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/a09dc4b6ed3343e2a7ef1506c0af7fd3] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=35.7 K 2024-11-20T06:40:08,530 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d5b6be1d75b49189840b18d26249838, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732084804674 2024-11-20T06:40:08,530 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting df3d54bc08bf48e88c4b88bb5e0c5bd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732084804674 2024-11-20T06:40:08,530 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2e4a61d7c544c7f87b8e6d4cea1628c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732084805857 2024-11-20T06:40:08,531 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting e242f1b8a712494e8437d5791e6fda21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732084805857 2024-11-20T06:40:08,531 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a09dc4b6ed3343e2a7ef1506c0af7fd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732084807013 2024-11-20T06:40:08,532 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 358c9ffaea384cc7866a2f277c8b9faa, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732084807013 2024-11-20T06:40:08,561 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:08,562 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/759ed9fba0284a4c968046edd7accf23 is 50, key is test_row_0/B:col10/1732084807069/Put/seqid=0 2024-11-20T06:40:08,571 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:08,574 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b127fc8d72434fc4abeb8cf7e0238277 is 50, key is test_row_0/A:col10/1732084807069/Put/seqid=0 2024-11-20T06:40:08,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741870_1046 (size=12527) 2024-11-20T06:40:08,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741871_1047 (size=12527) 2024-11-20T06:40:08,626 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/759ed9fba0284a4c968046edd7accf23 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/759ed9fba0284a4c968046edd7accf23 2024-11-20T06:40:08,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T06:40:08,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,634 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:40:08,642 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 759ed9fba0284a4c968046edd7accf23(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:08,642 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:08,642 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=13, startTime=1732084808527; duration=0sec 2024-11-20T06:40:08,642 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:08,642 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:08,642 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:08,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/66db5f3d3e914b3b9b57602ad72491a0 is 50, key is test_row_0/A:col10/1732084807127/Put/seqid=0 2024-11-20T06:40:08,644 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:08,645 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:08,645 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:08,645 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/dd898e992e8746ec9ffacb020e4ab0fb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7255d4ecd50d4bbf9893dfb33a437815, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b53c76cd62040a88820bc10d350df2f] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=35.7 K 2024-11-20T06:40:08,646 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting dd898e992e8746ec9ffacb020e4ab0fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732084804674 2024-11-20T06:40:08,647 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 7255d4ecd50d4bbf9893dfb33a437815, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732084805857 2024-11-20T06:40:08,648 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b53c76cd62040a88820bc10d350df2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732084807013 2024-11-20T06:40:08,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741872_1048 (size=12151) 2024-11-20T06:40:08,680 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#36 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:08,680 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/a0739a9ab41944baa22e0d4c1aba452e is 50, key is test_row_0/C:col10/1732084807069/Put/seqid=0 2024-11-20T06:40:08,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741873_1049 (size=12527) 2024-11-20T06:40:08,752 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/a0739a9ab41944baa22e0d4c1aba452e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a0739a9ab41944baa22e0d4c1aba452e 2024-11-20T06:40:08,766 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into a0739a9ab41944baa22e0d4c1aba452e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:08,766 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:08,766 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=13, startTime=1732084808527; duration=0sec 2024-11-20T06:40:08,766 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:08,766 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:09,038 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b127fc8d72434fc4abeb8cf7e0238277 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b127fc8d72434fc4abeb8cf7e0238277 2024-11-20T06:40:09,053 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into b127fc8d72434fc4abeb8cf7e0238277(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:09,053 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:09,053 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=13, startTime=1732084808527; duration=0sec 2024-11-20T06:40:09,053 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:09,053 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:09,077 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/66db5f3d3e914b3b9b57602ad72491a0 2024-11-20T06:40:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/9c7f93b5296e4703afaf496d514ccaae is 50, key is test_row_0/B:col10/1732084807127/Put/seqid=0 2024-11-20T06:40:09,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741874_1050 (size=12151) 2024-11-20T06:40:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:09,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:09,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084869373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T06:40:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084869374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084869376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084869378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084869383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084869495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084869495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084869495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084869495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084869504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,550 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/9c7f93b5296e4703afaf496d514ccaae 2024-11-20T06:40:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/863388e0e9c244adaf85f751ebff3ab0 is 50, key is test_row_0/C:col10/1732084807127/Put/seqid=0 2024-11-20T06:40:09,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741875_1051 (size=12151) 2024-11-20T06:40:09,593 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/863388e0e9c244adaf85f751ebff3ab0 2024-11-20T06:40:09,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/66db5f3d3e914b3b9b57602ad72491a0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/66db5f3d3e914b3b9b57602ad72491a0 2024-11-20T06:40:09,648 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/66db5f3d3e914b3b9b57602ad72491a0, entries=150, sequenceid=175, filesize=11.9 K 2024-11-20T06:40:09,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/9c7f93b5296e4703afaf496d514ccaae as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/9c7f93b5296e4703afaf496d514ccaae 2024-11-20T06:40:09,659 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/9c7f93b5296e4703afaf496d514ccaae, entries=150, sequenceid=175, filesize=11.9 K 2024-11-20T06:40:09,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/863388e0e9c244adaf85f751ebff3ab0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/863388e0e9c244adaf85f751ebff3ab0 2024-11-20T06:40:09,684 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/863388e0e9c244adaf85f751ebff3ab0, entries=150, sequenceid=175, filesize=11.9 K 2024-11-20T06:40:09,686 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 1052ms, sequenceid=175, compaction requested=false 2024-11-20T06:40:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-20T06:40:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-20T06:40:09,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-20T06:40:09,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4600 sec 2024-11-20T06:40:09,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.4700 sec 2024-11-20T06:40:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:09,723 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:09,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T06:40:09,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b65c54c6727b41c98fa2fa07084b0ae8 is 50, key is test_row_0/A:col10/1732084809374/Put/seqid=0 2024-11-20T06:40:09,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084869743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084869746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084869755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084869760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084869764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741876_1052 (size=12151) 2024-11-20T06:40:09,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b65c54c6727b41c98fa2fa07084b0ae8 2024-11-20T06:40:09,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/b783744ed81040c09ad88991f717155b is 50, key is test_row_0/B:col10/1732084809374/Put/seqid=0 2024-11-20T06:40:09,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741877_1053 (size=12151) 2024-11-20T06:40:09,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/b783744ed81040c09ad88991f717155b 2024-11-20T06:40:09,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084869864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084869864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/59c6bf18e2be4187867b251bbee97972 is 50, key is test_row_0/C:col10/1732084809374/Put/seqid=0 2024-11-20T06:40:09,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084869875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084869875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084869876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:09,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741878_1054 (size=12151) 2024-11-20T06:40:09,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/59c6bf18e2be4187867b251bbee97972 2024-11-20T06:40:09,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b65c54c6727b41c98fa2fa07084b0ae8 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b65c54c6727b41c98fa2fa07084b0ae8 2024-11-20T06:40:09,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b65c54c6727b41c98fa2fa07084b0ae8, entries=150, sequenceid=205, filesize=11.9 K 2024-11-20T06:40:09,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/b783744ed81040c09ad88991f717155b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/b783744ed81040c09ad88991f717155b 2024-11-20T06:40:09,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/b783744ed81040c09ad88991f717155b, entries=150, sequenceid=205, filesize=11.9 K 2024-11-20T06:40:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/59c6bf18e2be4187867b251bbee97972 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/59c6bf18e2be4187867b251bbee97972 2024-11-20T06:40:09,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/59c6bf18e2be4187867b251bbee97972, entries=150, sequenceid=205, filesize=11.9 K 2024-11-20T06:40:09,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 261ms, sequenceid=205, compaction requested=true 2024-11-20T06:40:09,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:09,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:09,985 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:09,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:09,985 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:09,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:09,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:09,987 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:09,987 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:09,987 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:09,987 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:09,987 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:09,987 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:09,987 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/759ed9fba0284a4c968046edd7accf23, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/9c7f93b5296e4703afaf496d514ccaae, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/b783744ed81040c09ad88991f717155b] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.0 K 2024-11-20T06:40:09,987 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b127fc8d72434fc4abeb8cf7e0238277, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/66db5f3d3e914b3b9b57602ad72491a0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b65c54c6727b41c98fa2fa07084b0ae8] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.0 K 2024-11-20T06:40:09,990 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting b127fc8d72434fc4abeb8cf7e0238277, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732084807013 2024-11-20T06:40:09,990 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 759ed9fba0284a4c968046edd7accf23, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732084807013 2024-11-20T06:40:09,991 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66db5f3d3e914b3b9b57602ad72491a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732084807117 2024-11-20T06:40:09,991 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c7f93b5296e4703afaf496d514ccaae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732084807117 2024-11-20T06:40:09,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:09,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:09,992 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting b65c54c6727b41c98fa2fa07084b0ae8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732084809374 2024-11-20T06:40:09,992 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b783744ed81040c09ad88991f717155b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732084809374 2024-11-20T06:40:10,021 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:10,022 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/03f771d6e8d1492f9c8358a7a5c8157c is 50, key is test_row_0/A:col10/1732084809374/Put/seqid=0 2024-11-20T06:40:10,029 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:10,030 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/107841891c4240e2af3be5bb26221915 is 50, key is test_row_0/B:col10/1732084809374/Put/seqid=0 2024-11-20T06:40:10,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741879_1055 (size=12629) 2024-11-20T06:40:10,078 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:10,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:40:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:10,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741880_1056 (size=12629) 2024-11-20T06:40:10,086 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/03f771d6e8d1492f9c8358a7a5c8157c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/03f771d6e8d1492f9c8358a7a5c8157c 2024-11-20T06:40:10,098 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/107841891c4240e2af3be5bb26221915 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/107841891c4240e2af3be5bb26221915 2024-11-20T06:40:10,099 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 03f771d6e8d1492f9c8358a7a5c8157c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:10,099 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:10,099 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=13, startTime=1732084809985; duration=0sec 2024-11-20T06:40:10,099 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:10,099 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:10,100 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:10,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/01973a32b6ad4db99d03d3aa02fca40a is 50, key is test_row_0/A:col10/1732084809729/Put/seqid=0 2024-11-20T06:40:10,101 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:10,101 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:10,101 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:10,102 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a0739a9ab41944baa22e0d4c1aba452e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/863388e0e9c244adaf85f751ebff3ab0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/59c6bf18e2be4187867b251bbee97972] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.0 K 2024-11-20T06:40:10,105 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0739a9ab41944baa22e0d4c1aba452e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1732084807013 2024-11-20T06:40:10,108 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 107841891c4240e2af3be5bb26221915(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:10,108 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:10,108 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=13, startTime=1732084809985; duration=0sec 2024-11-20T06:40:10,108 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:10,108 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:10,109 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 863388e0e9c244adaf85f751ebff3ab0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732084807117 2024-11-20T06:40:10,110 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59c6bf18e2be4187867b251bbee97972, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732084809374 2024-11-20T06:40:10,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741881_1057 (size=12151) 2024-11-20T06:40:10,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/01973a32b6ad4db99d03d3aa02fca40a 2024-11-20T06:40:10,149 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:10,150 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/ce06e771defa4c8ea3b41cef0938852b is 50, key is test_row_0/C:col10/1732084809374/Put/seqid=0 2024-11-20T06:40:10,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/88d39d43acb640a7899895165c672fea is 50, key is test_row_0/B:col10/1732084809729/Put/seqid=0 2024-11-20T06:40:10,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084870151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084870156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084870156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084870160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084870161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741882_1058 (size=12629) 2024-11-20T06:40:10,188 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/ce06e771defa4c8ea3b41cef0938852b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/ce06e771defa4c8ea3b41cef0938852b 2024-11-20T06:40:10,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741883_1059 (size=12151) 2024-11-20T06:40:10,198 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into ce06e771defa4c8ea3b41cef0938852b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:10,198 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:10,198 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=13, startTime=1732084809986; duration=0sec 2024-11-20T06:40:10,198 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:10,198 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:10,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/88d39d43acb640a7899895165c672fea 2024-11-20T06:40:10,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/be5b80a573ff4625af64073442218c9a is 50, key is test_row_0/C:col10/1732084809729/Put/seqid=0 2024-11-20T06:40:10,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741884_1060 (size=12151) 2024-11-20T06:40:10,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/be5b80a573ff4625af64073442218c9a 2024-11-20T06:40:10,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/01973a32b6ad4db99d03d3aa02fca40a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/01973a32b6ad4db99d03d3aa02fca40a 2024-11-20T06:40:10,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/01973a32b6ad4db99d03d3aa02fca40a, entries=150, sequenceid=216, filesize=11.9 K 2024-11-20T06:40:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084870264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/88d39d43acb640a7899895165c672fea as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/88d39d43acb640a7899895165c672fea 2024-11-20T06:40:10,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084870274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084870274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084870275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084870275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/88d39d43acb640a7899895165c672fea, entries=150, sequenceid=216, filesize=11.9 K 2024-11-20T06:40:10,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/be5b80a573ff4625af64073442218c9a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/be5b80a573ff4625af64073442218c9a 2024-11-20T06:40:10,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/be5b80a573ff4625af64073442218c9a, entries=150, sequenceid=216, filesize=11.9 K 2024-11-20T06:40:10,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 220ms, sequenceid=216, compaction requested=false 2024-11-20T06:40:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:10,478 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:10,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T06:40:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:10,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b14d4fbf72dd4f52bcef007d6ec82505 is 50, key is test_row_0/A:col10/1732084810150/Put/seqid=0 2024-11-20T06:40:10,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084870497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084870497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084870497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084870499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084870500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741885_1061 (size=14541) 2024-11-20T06:40:10,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b14d4fbf72dd4f52bcef007d6ec82505 2024-11-20T06:40:10,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/21d1b46b964943849a8d54c267c130b9 is 50, key is test_row_0/B:col10/1732084810150/Put/seqid=0 2024-11-20T06:40:10,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741886_1062 (size=12151) 2024-11-20T06:40:10,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/21d1b46b964943849a8d54c267c130b9 2024-11-20T06:40:10,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/aff9d5bc5dbd4b5dab396c9e3095e614 is 50, key is test_row_0/C:col10/1732084810150/Put/seqid=0 2024-11-20T06:40:10,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741887_1063 (size=12151) 2024-11-20T06:40:10,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/aff9d5bc5dbd4b5dab396c9e3095e614 2024-11-20T06:40:10,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084870615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084870615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084870615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084870618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084870618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b14d4fbf72dd4f52bcef007d6ec82505 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b14d4fbf72dd4f52bcef007d6ec82505 2024-11-20T06:40:10,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b14d4fbf72dd4f52bcef007d6ec82505, entries=200, sequenceid=245, filesize=14.2 K 2024-11-20T06:40:10,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/21d1b46b964943849a8d54c267c130b9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/21d1b46b964943849a8d54c267c130b9 2024-11-20T06:40:10,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/21d1b46b964943849a8d54c267c130b9, entries=150, sequenceid=245, filesize=11.9 K 2024-11-20T06:40:10,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/aff9d5bc5dbd4b5dab396c9e3095e614 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/aff9d5bc5dbd4b5dab396c9e3095e614 2024-11-20T06:40:10,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/aff9d5bc5dbd4b5dab396c9e3095e614, entries=150, sequenceid=245, filesize=11.9 K 2024-11-20T06:40:10,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 205ms, sequenceid=245, compaction requested=true 2024-11-20T06:40:10,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:10,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:10,690 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:10,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:10,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:10,690 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:10,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:10,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:10,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:10,691 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:10,691 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:10,691 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:10,691 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:10,691 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:10,691 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:10,692 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/03f771d6e8d1492f9c8358a7a5c8157c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/01973a32b6ad4db99d03d3aa02fca40a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b14d4fbf72dd4f52bcef007d6ec82505] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=38.4 K 2024-11-20T06:40:10,692 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/107841891c4240e2af3be5bb26221915, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/88d39d43acb640a7899895165c672fea, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/21d1b46b964943849a8d54c267c130b9] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.1 K 2024-11-20T06:40:10,692 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 107841891c4240e2af3be5bb26221915, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732084809374 2024-11-20T06:40:10,692 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03f771d6e8d1492f9c8358a7a5c8157c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732084809374 2024-11-20T06:40:10,692 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01973a32b6ad4db99d03d3aa02fca40a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732084809729 2024-11-20T06:40:10,692 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 88d39d43acb640a7899895165c672fea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732084809729 2024-11-20T06:40:10,693 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting b14d4fbf72dd4f52bcef007d6ec82505, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732084810133 2024-11-20T06:40:10,693 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d1b46b964943849a8d54c267c130b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732084810133 2024-11-20T06:40:10,721 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:10,721 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/6dc099192b5540d589a7a5a715da382d is 50, key is test_row_0/B:col10/1732084810150/Put/seqid=0 2024-11-20T06:40:10,741 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:10,742 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/98539334b7e94b5c82374997e004bb3e is 50, key is test_row_0/A:col10/1732084810150/Put/seqid=0 2024-11-20T06:40:10,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741888_1064 (size=12731) 2024-11-20T06:40:10,794 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/6dc099192b5540d589a7a5a715da382d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/6dc099192b5540d589a7a5a715da382d 2024-11-20T06:40:10,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741889_1065 (size=12731) 2024-11-20T06:40:10,809 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 6dc099192b5540d589a7a5a715da382d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:10,809 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:10,809 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=13, startTime=1732084810690; duration=0sec 2024-11-20T06:40:10,809 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:10,809 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:10,810 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:10,813 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:10,814 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:10,814 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:10,814 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/ce06e771defa4c8ea3b41cef0938852b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/be5b80a573ff4625af64073442218c9a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/aff9d5bc5dbd4b5dab396c9e3095e614] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.1 K 2024-11-20T06:40:10,815 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ce06e771defa4c8ea3b41cef0938852b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732084809374 2024-11-20T06:40:10,816 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting be5b80a573ff4625af64073442218c9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732084809729 2024-11-20T06:40:10,817 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting aff9d5bc5dbd4b5dab396c9e3095e614, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732084810133 2024-11-20T06:40:10,820 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/98539334b7e94b5c82374997e004bb3e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/98539334b7e94b5c82374997e004bb3e 2024-11-20T06:40:10,829 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:10,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:40:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:10,834 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 98539334b7e94b5c82374997e004bb3e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:10,834 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:10,834 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=13, startTime=1732084810690; duration=0sec 2024-11-20T06:40:10,834 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:10,834 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:10,840 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:10,840 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/44046fedceef4ddda333a41bc502cda2 is 50, key is test_row_0/C:col10/1732084810150/Put/seqid=0 2024-11-20T06:40:10,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/ea7161d0609146ca94381e77f4fe44ae is 50, key is test_row_0/A:col10/1732084810827/Put/seqid=0 2024-11-20T06:40:10,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741890_1066 (size=12731) 2024-11-20T06:40:10,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741891_1067 (size=14691) 2024-11-20T06:40:10,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084870902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084870902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084870905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084870905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:10,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084870905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084871015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084871015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084871015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084871025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084871025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084871225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084871225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084871226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084871234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084871234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/ea7161d0609146ca94381e77f4fe44ae 2024-11-20T06:40:11,288 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/44046fedceef4ddda333a41bc502cda2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/44046fedceef4ddda333a41bc502cda2 2024-11-20T06:40:11,300 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 44046fedceef4ddda333a41bc502cda2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:11,300 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:11,300 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=13, startTime=1732084810690; duration=0sec 2024-11-20T06:40:11,300 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:11,300 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:11,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/ff69632a0f8f4832a4892b32b0f39864 is 50, key is test_row_0/B:col10/1732084810827/Put/seqid=0 2024-11-20T06:40:11,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741892_1068 (size=12301) 2024-11-20T06:40:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T06:40:11,404 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:11,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-20T06:40:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T06:40:11,424 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:11,427 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:11,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T06:40:11,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084871534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084871544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084871545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084871545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084871546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:11,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:11,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T06:40:11,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:11,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:11,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/ff69632a0f8f4832a4892b32b0f39864 2024-11-20T06:40:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T06:40:11,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T06:40:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:11,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:11,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/bda2a4ef36b5416295d74da56ce5f10b is 50, key is test_row_0/C:col10/1732084810827/Put/seqid=0 2024-11-20T06:40:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:11,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741893_1069 (size=12301) 2024-11-20T06:40:11,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T06:40:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:11,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:11,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T06:40:12,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084872044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:12,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T06:40:12,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:12,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:12,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:12,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:12,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:12,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084872055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:12,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:12,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:12,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:12,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084872055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:12,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084872055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:12,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084872064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:12,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/bda2a4ef36b5416295d74da56ce5f10b 2024-11-20T06:40:12,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/ea7161d0609146ca94381e77f4fe44ae as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/ea7161d0609146ca94381e77f4fe44ae 2024-11-20T06:40:12,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T06:40:12,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:12,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:12,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:12,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:12,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:12,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/ea7161d0609146ca94381e77f4fe44ae, entries=200, sequenceid=260, filesize=14.3 K 2024-11-20T06:40:12,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/ff69632a0f8f4832a4892b32b0f39864 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ff69632a0f8f4832a4892b32b0f39864 2024-11-20T06:40:12,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ff69632a0f8f4832a4892b32b0f39864, entries=150, sequenceid=260, filesize=12.0 K 2024-11-20T06:40:12,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/bda2a4ef36b5416295d74da56ce5f10b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/bda2a4ef36b5416295d74da56ce5f10b 2024-11-20T06:40:12,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/bda2a4ef36b5416295d74da56ce5f10b, entries=150, sequenceid=260, filesize=12.0 K 2024-11-20T06:40:12,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 1437ms, sequenceid=260, compaction requested=false 2024-11-20T06:40:12,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:12,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T06:40:12,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:12,367 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T06:40:12,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/656b476ad6ce44a58922df859d48d2db is 50, key is test_row_0/A:col10/1732084810901/Put/seqid=0 2024-11-20T06:40:12,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741894_1070 (size=12301) 2024-11-20T06:40:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T06:40:12,847 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/656b476ad6ce44a58922df859d48d2db 2024-11-20T06:40:12,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/dcda523c30044384a057e53911555842 is 50, key is test_row_0/B:col10/1732084810901/Put/seqid=0 2024-11-20T06:40:12,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741895_1071 (size=12301) 2024-11-20T06:40:12,905 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/dcda523c30044384a057e53911555842 2024-11-20T06:40:12,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/7e83dabd83b84bc69e7caf82a8ebfe31 is 50, key is test_row_0/C:col10/1732084810901/Put/seqid=0 2024-11-20T06:40:12,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741896_1072 (size=12301) 2024-11-20T06:40:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:13,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084873076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084873077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084873077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084873078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084873084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,093 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in 8b50cf3b5c22a8ba480f30fa5ecd19ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:13,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084873194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084873194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084873195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084873195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,323 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/7e83dabd83b84bc69e7caf82a8ebfe31 2024-11-20T06:40:13,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/656b476ad6ce44a58922df859d48d2db as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/656b476ad6ce44a58922df859d48d2db 2024-11-20T06:40:13,343 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/656b476ad6ce44a58922df859d48d2db, entries=150, sequenceid=286, filesize=12.0 K 2024-11-20T06:40:13,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/dcda523c30044384a057e53911555842 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/dcda523c30044384a057e53911555842 2024-11-20T06:40:13,352 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/dcda523c30044384a057e53911555842, entries=150, sequenceid=286, filesize=12.0 K 2024-11-20T06:40:13,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/7e83dabd83b84bc69e7caf82a8ebfe31 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7e83dabd83b84bc69e7caf82a8ebfe31 2024-11-20T06:40:13,362 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7e83dabd83b84bc69e7caf82a8ebfe31, entries=150, sequenceid=286, filesize=12.0 K 2024-11-20T06:40:13,364 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 996ms, sequenceid=286, compaction requested=true 2024-11-20T06:40:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-20T06:40:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-20T06:40:13,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T06:40:13,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9380 sec 2024-11-20T06:40:13,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.9690 sec 2024-11-20T06:40:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:13,407 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:13,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T06:40:13,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/1acde39aff7a461380f2250dec6c410e is 50, key is test_row_0/A:col10/1732084813075/Put/seqid=0 2024-11-20T06:40:13,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741897_1073 (size=17181) 2024-11-20T06:40:13,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084873452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084873456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084873458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084873458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T06:40:13,564 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:13,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-20T06:40:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T06:40:13,571 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:13,572 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:13,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:13,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084873574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084873575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084873575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084873575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T06:40:13,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T06:40:13,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:13,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:13,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:13,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:13,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:13,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084873783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084873784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084873784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084873784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:13,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/1acde39aff7a461380f2250dec6c410e 2024-11-20T06:40:13,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/128a8d545045409f95c7fcd508179510 is 50, key is test_row_0/B:col10/1732084813075/Put/seqid=0 2024-11-20T06:40:13,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741898_1074 (size=12301) 2024-11-20T06:40:13,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T06:40:13,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:13,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:13,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:13,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:13,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T06:40:14,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T06:40:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:14,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:14,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084874094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084874094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084874094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084874094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T06:40:14,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T06:40:14,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:14,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:14,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/128a8d545045409f95c7fcd508179510 2024-11-20T06:40:14,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/325e33bfec354cd881fd99075370929f is 50, key is test_row_0/C:col10/1732084813075/Put/seqid=0 2024-11-20T06:40:14,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741899_1075 (size=12301) 2024-11-20T06:40:14,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/325e33bfec354cd881fd99075370929f 2024-11-20T06:40:14,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/1acde39aff7a461380f2250dec6c410e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1acde39aff7a461380f2250dec6c410e 2024-11-20T06:40:14,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1acde39aff7a461380f2250dec6c410e, entries=250, sequenceid=299, filesize=16.8 K 2024-11-20T06:40:14,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/128a8d545045409f95c7fcd508179510 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/128a8d545045409f95c7fcd508179510 2024-11-20T06:40:14,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T06:40:14,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:14,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:14,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:14,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:14,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:14,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/128a8d545045409f95c7fcd508179510, entries=150, sequenceid=299, filesize=12.0 K 2024-11-20T06:40:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/325e33bfec354cd881fd99075370929f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/325e33bfec354cd881fd99075370929f 2024-11-20T06:40:14,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/325e33bfec354cd881fd99075370929f, entries=150, sequenceid=299, filesize=12.0 K 2024-11-20T06:40:14,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 979ms, sequenceid=299, compaction requested=true 2024-11-20T06:40:14,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:14,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:14,387 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:14,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:14,387 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:14,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:14,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:14,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:14,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:14,389 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56904 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:14,389 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:14,389 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:14,389 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:14,389 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:14,389 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:14,389 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/6dc099192b5540d589a7a5a715da382d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ff69632a0f8f4832a4892b32b0f39864, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/dcda523c30044384a057e53911555842, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/128a8d545045409f95c7fcd508179510] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=48.5 K 2024-11-20T06:40:14,389 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/98539334b7e94b5c82374997e004bb3e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/ea7161d0609146ca94381e77f4fe44ae, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/656b476ad6ce44a58922df859d48d2db, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1acde39aff7a461380f2250dec6c410e] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=55.6 K 2024-11-20T06:40:14,389 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc099192b5540d589a7a5a715da382d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732084810133 2024-11-20T06:40:14,389 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98539334b7e94b5c82374997e004bb3e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732084810133 2024-11-20T06:40:14,390 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ff69632a0f8f4832a4892b32b0f39864, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732084810827 2024-11-20T06:40:14,390 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea7161d0609146ca94381e77f4fe44ae, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732084810498 2024-11-20T06:40:14,390 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 656b476ad6ce44a58922df859d48d2db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732084810887 2024-11-20T06:40:14,390 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting dcda523c30044384a057e53911555842, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732084810887 2024-11-20T06:40:14,391 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 128a8d545045409f95c7fcd508179510, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732084813075 2024-11-20T06:40:14,391 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1acde39aff7a461380f2250dec6c410e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732084813069 2024-11-20T06:40:14,406 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:14,406 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:14,406 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/afc443fdc68e42958cabdd3e1ac72a4b is 50, key is test_row_0/B:col10/1732084813075/Put/seqid=0 2024-11-20T06:40:14,407 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/2aa0a5bbeec448ccbea52cfa49e7f566 is 50, key is test_row_0/A:col10/1732084813075/Put/seqid=0 2024-11-20T06:40:14,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741901_1077 (size=13017) 2024-11-20T06:40:14,432 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/2aa0a5bbeec448ccbea52cfa49e7f566 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/2aa0a5bbeec448ccbea52cfa49e7f566 2024-11-20T06:40:14,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741900_1076 (size=13017) 2024-11-20T06:40:14,445 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 2aa0a5bbeec448ccbea52cfa49e7f566(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:14,446 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:14,447 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=12, startTime=1732084814387; duration=0sec 2024-11-20T06:40:14,447 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:14,447 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:14,447 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:14,449 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/afc443fdc68e42958cabdd3e1ac72a4b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/afc443fdc68e42958cabdd3e1ac72a4b 2024-11-20T06:40:14,450 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:14,450 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:14,450 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:14,450 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/44046fedceef4ddda333a41bc502cda2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/bda2a4ef36b5416295d74da56ce5f10b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7e83dabd83b84bc69e7caf82a8ebfe31, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/325e33bfec354cd881fd99075370929f] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=48.5 K 2024-11-20T06:40:14,451 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44046fedceef4ddda333a41bc502cda2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732084810133 2024-11-20T06:40:14,452 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting bda2a4ef36b5416295d74da56ce5f10b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732084810827 2024-11-20T06:40:14,456 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e83dabd83b84bc69e7caf82a8ebfe31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732084810887 2024-11-20T06:40:14,457 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 325e33bfec354cd881fd99075370929f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732084813075 2024-11-20T06:40:14,460 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into afc443fdc68e42958cabdd3e1ac72a4b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:14,460 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:14,460 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=12, startTime=1732084814387; duration=0sec 2024-11-20T06:40:14,460 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:14,460 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:14,478 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:14,479 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/0fa5dbebf7174160947d43b7152fb742 is 50, key is test_row_0/C:col10/1732084813075/Put/seqid=0 2024-11-20T06:40:14,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T06:40:14,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:14,508 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T06:40:14,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/8bbcccb014ee432892116a59c78adb68 is 50, key is test_row_0/A:col10/1732084813453/Put/seqid=0 2024-11-20T06:40:14,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741902_1078 (size=13017) 2024-11-20T06:40:14,533 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/0fa5dbebf7174160947d43b7152fb742 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0fa5dbebf7174160947d43b7152fb742 2024-11-20T06:40:14,543 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 0fa5dbebf7174160947d43b7152fb742(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:14,543 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:14,544 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=12, startTime=1732084814387; duration=0sec 2024-11-20T06:40:14,544 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:14,544 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:14,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741903_1079 (size=12301) 2024-11-20T06:40:14,552 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/8bbcccb014ee432892116a59c78adb68 2024-11-20T06:40:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/fd3ecf6d045649f69af75ba7be756f26 is 50, key is test_row_0/B:col10/1732084813453/Put/seqid=0 2024-11-20T06:40:14,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741904_1080 (size=12301) 2024-11-20T06:40:14,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:14,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084874616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084874618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084874618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084874621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T06:40:14,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084874723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084874723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084874734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084874734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084874934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084874934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084874944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084874944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:14,997 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/fd3ecf6d045649f69af75ba7be756f26 2024-11-20T06:40:15,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/9427ceb96f964ac6a04ca1a2ca969e47 is 50, key is test_row_0/C:col10/1732084813453/Put/seqid=0 2024-11-20T06:40:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741905_1081 (size=12301) 2024-11-20T06:40:15,025 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/9427ceb96f964ac6a04ca1a2ca969e47 2024-11-20T06:40:15,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/8bbcccb014ee432892116a59c78adb68 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8bbcccb014ee432892116a59c78adb68 2024-11-20T06:40:15,042 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8bbcccb014ee432892116a59c78adb68, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T06:40:15,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/fd3ecf6d045649f69af75ba7be756f26 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/fd3ecf6d045649f69af75ba7be756f26 2024-11-20T06:40:15,051 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/fd3ecf6d045649f69af75ba7be756f26, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T06:40:15,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/9427ceb96f964ac6a04ca1a2ca969e47 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/9427ceb96f964ac6a04ca1a2ca969e47 2024-11-20T06:40:15,061 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/9427ceb96f964ac6a04ca1a2ca969e47, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T06:40:15,062 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 555ms, sequenceid=326, compaction requested=false 2024-11-20T06:40:15,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:15,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:15,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-20T06:40:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-20T06:40:15,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T06:40:15,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4930 sec 2024-11-20T06:40:15,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5030 sec 2024-11-20T06:40:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:15,106 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:15,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T06:40:15,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/648ae321fb9c44ffbf906c9f69c22b51 is 50, key is test_row_0/A:col10/1732084815105/Put/seqid=0 2024-11-20T06:40:15,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741906_1082 (size=12301) 2024-11-20T06:40:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084875246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084875248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084875250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084875254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084875254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084875365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084875365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084875365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/648ae321fb9c44ffbf906c9f69c22b51 2024-11-20T06:40:15,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084875574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084875574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084875574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/42625d0e37024817b34f671b9ca57318 is 50, key is test_row_0/B:col10/1732084815105/Put/seqid=0 2024-11-20T06:40:15,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741907_1083 (size=12301) 2024-11-20T06:40:15,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/42625d0e37024817b34f671b9ca57318 2024-11-20T06:40:15,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/6d505f61b36d48da9bcad7b3e5082a6a is 50, key is test_row_0/C:col10/1732084815105/Put/seqid=0 2024-11-20T06:40:15,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741908_1084 (size=12301) 2024-11-20T06:40:15,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/6d505f61b36d48da9bcad7b3e5082a6a 2024-11-20T06:40:15,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/648ae321fb9c44ffbf906c9f69c22b51 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/648ae321fb9c44ffbf906c9f69c22b51 2024-11-20T06:40:15,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/648ae321fb9c44ffbf906c9f69c22b51, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T06:40:15,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/42625d0e37024817b34f671b9ca57318 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/42625d0e37024817b34f671b9ca57318 2024-11-20T06:40:15,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/42625d0e37024817b34f671b9ca57318, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T06:40:15,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/6d505f61b36d48da9bcad7b3e5082a6a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6d505f61b36d48da9bcad7b3e5082a6a 2024-11-20T06:40:15,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6d505f61b36d48da9bcad7b3e5082a6a, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T06:40:15,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 573ms, sequenceid=339, compaction requested=true 2024-11-20T06:40:15,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:15,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:15,679 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:15,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:15,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:15,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:15,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:15,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T06:40:15,680 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:15,681 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:15,681 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:15,681 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:15,681 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:15,682 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/2aa0a5bbeec448ccbea52cfa49e7f566, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8bbcccb014ee432892116a59c78adb68, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/648ae321fb9c44ffbf906c9f69c22b51] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.7 K 2024-11-20T06:40:15,682 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:15,682 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:15,682 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/afc443fdc68e42958cabdd3e1ac72a4b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/fd3ecf6d045649f69af75ba7be756f26, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/42625d0e37024817b34f671b9ca57318] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.7 K 2024-11-20T06:40:15,682 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aa0a5bbeec448ccbea52cfa49e7f566, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732084813075 2024-11-20T06:40:15,682 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting afc443fdc68e42958cabdd3e1ac72a4b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732084813075 2024-11-20T06:40:15,683 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bbcccb014ee432892116a59c78adb68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732084813447 2024-11-20T06:40:15,683 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting fd3ecf6d045649f69af75ba7be756f26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732084813447 2024-11-20T06:40:15,683 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 648ae321fb9c44ffbf906c9f69c22b51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084814612 2024-11-20T06:40:15,683 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 42625d0e37024817b34f671b9ca57318, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084814612 2024-11-20T06:40:15,709 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:15,710 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:15,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/926e558898d3463b9582ca16c66c7c41 is 50, key is test_row_0/A:col10/1732084815105/Put/seqid=0 2024-11-20T06:40:15,710 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/4afb7d7136874ffc920c1e2d3eaaca22 is 50, key is test_row_0/B:col10/1732084815105/Put/seqid=0 2024-11-20T06:40:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T06:40:15,714 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:15,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-20T06:40:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T06:40:15,718 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:15,720 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:15,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:15,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741909_1085 (size=13119) 2024-11-20T06:40:15,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741910_1086 (size=13119) 2024-11-20T06:40:15,740 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/926e558898d3463b9582ca16c66c7c41 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/926e558898d3463b9582ca16c66c7c41 2024-11-20T06:40:15,751 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 926e558898d3463b9582ca16c66c7c41(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:15,751 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:15,751 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=13, startTime=1732084815679; duration=0sec 2024-11-20T06:40:15,751 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:15,751 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:15,751 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:15,753 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:15,753 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:15,754 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:15,754 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0fa5dbebf7174160947d43b7152fb742, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/9427ceb96f964ac6a04ca1a2ca969e47, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6d505f61b36d48da9bcad7b3e5082a6a] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.7 K 2024-11-20T06:40:15,755 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fa5dbebf7174160947d43b7152fb742, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732084813075 2024-11-20T06:40:15,755 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9427ceb96f964ac6a04ca1a2ca969e47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732084813447 2024-11-20T06:40:15,758 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d505f61b36d48da9bcad7b3e5082a6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084814612 2024-11-20T06:40:15,766 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:15,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T06:40:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:15,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/849b9d396c2a488e92e072d175d19436 is 50, key is test_row_0/A:col10/1732084815764/Put/seqid=0 2024-11-20T06:40:15,782 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:15,784 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/473e126ae93d4b12a92fe89772065ce4 is 50, key is test_row_0/C:col10/1732084815105/Put/seqid=0 2024-11-20T06:40:15,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084875795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084875798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741912_1088 (size=13119) 2024-11-20T06:40:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T06:40:15,825 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/473e126ae93d4b12a92fe89772065ce4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/473e126ae93d4b12a92fe89772065ce4 2024-11-20T06:40:15,832 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 473e126ae93d4b12a92fe89772065ce4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:15,832 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:15,833 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=13, startTime=1732084815679; duration=0sec 2024-11-20T06:40:15,833 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:15,833 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:15,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741911_1087 (size=12301) 2024-11-20T06:40:15,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/849b9d396c2a488e92e072d175d19436 2024-11-20T06:40:15,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/569271cb1cc94c818e3bf0e37fa1c076 is 50, key is test_row_0/B:col10/1732084815764/Put/seqid=0 2024-11-20T06:40:15,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T06:40:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:15,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:15,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:15,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084875885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084875885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084875885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741913_1089 (size=12301) 2024-11-20T06:40:15,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084875904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:15,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084875914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T06:40:16,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:16,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T06:40:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084876113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084876124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,136 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/4afb7d7136874ffc920c1e2d3eaaca22 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4afb7d7136874ffc920c1e2d3eaaca22 2024-11-20T06:40:16,143 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 4afb7d7136874ffc920c1e2d3eaaca22(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:16,143 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:16,143 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=13, startTime=1732084815679; duration=0sec 2024-11-20T06:40:16,143 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:16,143 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:16,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T06:40:16,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:16,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/569271cb1cc94c818e3bf0e37fa1c076 2024-11-20T06:40:16,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/6fa72b0c0efe47c9903aaf46dcf85ece is 50, key is test_row_0/C:col10/1732084815764/Put/seqid=0 2024-11-20T06:40:16,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741914_1090 (size=12301) 2024-11-20T06:40:16,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/6fa72b0c0efe47c9903aaf46dcf85ece 2024-11-20T06:40:16,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T06:40:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/849b9d396c2a488e92e072d175d19436 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/849b9d396c2a488e92e072d175d19436 2024-11-20T06:40:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T06:40:16,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/849b9d396c2a488e92e072d175d19436, entries=150, sequenceid=365, filesize=12.0 K 2024-11-20T06:40:16,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/569271cb1cc94c818e3bf0e37fa1c076 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/569271cb1cc94c818e3bf0e37fa1c076 2024-11-20T06:40:16,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/569271cb1cc94c818e3bf0e37fa1c076, entries=150, sequenceid=365, filesize=12.0 K 2024-11-20T06:40:16,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/6fa72b0c0efe47c9903aaf46dcf85ece as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6fa72b0c0efe47c9903aaf46dcf85ece 2024-11-20T06:40:16,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084876395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084876395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084876395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6fa72b0c0efe47c9903aaf46dcf85ece, entries=150, sequenceid=365, filesize=12.0 K 2024-11-20T06:40:16,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 637ms, sequenceid=365, compaction requested=false 2024-11-20T06:40:16,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:16,435 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:16,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T06:40:16,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/0664e77badde441faf735f20eafcdcc7 is 50, key is test_row_0/A:col10/1732084815788/Put/seqid=0 2024-11-20T06:40:16,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T06:40:16,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:16,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741915_1091 (size=12301) 2024-11-20T06:40:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084876520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084876521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084876624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084876634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T06:40:16,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:16,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T06:40:16,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:16,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084876834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084876843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T06:40:16,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/0664e77badde441faf735f20eafcdcc7 2024-11-20T06:40:16,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/2f7968b63e7946108b99d9a3bb9a8c1e is 50, key is test_row_0/B:col10/1732084815788/Put/seqid=0 2024-11-20T06:40:16,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741916_1092 (size=12301) 2024-11-20T06:40:16,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/2f7968b63e7946108b99d9a3bb9a8c1e 2024-11-20T06:40:16,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/0c97c6bfa35b42869d99e8ec47d0f910 is 50, key is test_row_0/C:col10/1732084815788/Put/seqid=0 2024-11-20T06:40:16,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741917_1093 (size=12301) 2024-11-20T06:40:16,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/0c97c6bfa35b42869d99e8ec47d0f910 2024-11-20T06:40:16,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/0664e77badde441faf735f20eafcdcc7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0664e77badde441faf735f20eafcdcc7 2024-11-20T06:40:16,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T06:40:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:16,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0664e77badde441faf735f20eafcdcc7, entries=150, sequenceid=379, filesize=12.0 K 2024-11-20T06:40:16,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/2f7968b63e7946108b99d9a3bb9a8c1e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/2f7968b63e7946108b99d9a3bb9a8c1e 2024-11-20T06:40:16,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/2f7968b63e7946108b99d9a3bb9a8c1e, entries=150, sequenceid=379, filesize=12.0 K 2024-11-20T06:40:16,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/0c97c6bfa35b42869d99e8ec47d0f910 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0c97c6bfa35b42869d99e8ec47d0f910 2024-11-20T06:40:16,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0c97c6bfa35b42869d99e8ec47d0f910, entries=150, sequenceid=379, filesize=12.0 K 2024-11-20T06:40:16,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 555ms, sequenceid=379, compaction requested=true 2024-11-20T06:40:16,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:16,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:16,991 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:16,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:16,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:16,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:16,991 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:16,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:16,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:16,992 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:16,992 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:16,993 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,993 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4afb7d7136874ffc920c1e2d3eaaca22, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/569271cb1cc94c818e3bf0e37fa1c076, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/2f7968b63e7946108b99d9a3bb9a8c1e] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.8 K 2024-11-20T06:40:16,993 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:16,993 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:16,993 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:16,993 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/926e558898d3463b9582ca16c66c7c41, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/849b9d396c2a488e92e072d175d19436, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0664e77badde441faf735f20eafcdcc7] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.8 K 2024-11-20T06:40:16,994 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 926e558898d3463b9582ca16c66c7c41, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084814612 2024-11-20T06:40:16,994 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 4afb7d7136874ffc920c1e2d3eaaca22, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084814612 2024-11-20T06:40:16,996 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 569271cb1cc94c818e3bf0e37fa1c076, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732084815211 2024-11-20T06:40:16,996 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 849b9d396c2a488e92e072d175d19436, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732084815211 2024-11-20T06:40:16,998 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f7968b63e7946108b99d9a3bb9a8c1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732084815788 2024-11-20T06:40:16,999 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0664e77badde441faf735f20eafcdcc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732084815788 2024-11-20T06:40:17,025 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:17,025 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:17,026 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/af04226be0fc43ed92ccba7d4ec1f975 is 50, key is test_row_0/A:col10/1732084815788/Put/seqid=0 2024-11-20T06:40:17,026 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/f4f5d6a6b46841b5b8baf74331ee8db1 is 50, key is test_row_0/B:col10/1732084815788/Put/seqid=0 2024-11-20T06:40:17,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741918_1094 (size=13221) 2024-11-20T06:40:17,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741919_1095 (size=13221) 2024-11-20T06:40:17,041 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/f4f5d6a6b46841b5b8baf74331ee8db1 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/f4f5d6a6b46841b5b8baf74331ee8db1 2024-11-20T06:40:17,048 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into f4f5d6a6b46841b5b8baf74331ee8db1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:17,048 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:17,048 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=13, startTime=1732084816991; duration=0sec 2024-11-20T06:40:17,048 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:17,048 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:17,048 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:17,050 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:17,050 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:17,050 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:17,050 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/473e126ae93d4b12a92fe89772065ce4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6fa72b0c0efe47c9903aaf46dcf85ece, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0c97c6bfa35b42869d99e8ec47d0f910] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.8 K 2024-11-20T06:40:17,051 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 473e126ae93d4b12a92fe89772065ce4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084814612 2024-11-20T06:40:17,051 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fa72b0c0efe47c9903aaf46dcf85ece, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732084815211 2024-11-20T06:40:17,052 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c97c6bfa35b42869d99e8ec47d0f910, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732084815788 2024-11-20T06:40:17,063 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:17,064 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/ceaff7f9a5424bfda87f8a4877a631e7 is 50, key is test_row_0/C:col10/1732084815788/Put/seqid=0 2024-11-20T06:40:17,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741920_1096 (size=13221) 2024-11-20T06:40:17,084 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/ceaff7f9a5424bfda87f8a4877a631e7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/ceaff7f9a5424bfda87f8a4877a631e7 2024-11-20T06:40:17,105 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into ceaff7f9a5424bfda87f8a4877a631e7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:17,105 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:17,105 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=13, startTime=1732084816991; duration=0sec 2024-11-20T06:40:17,105 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:17,106 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:17,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T06:40:17,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:17,112 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T06:40:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/41d859fc1c0245f5a36bc74b71f97d2a is 50, key is test_row_0/A:col10/1732084816518/Put/seqid=0 2024-11-20T06:40:17,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741921_1097 (size=12301) 2024-11-20T06:40:17,144 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/41d859fc1c0245f5a36bc74b71f97d2a 2024-11-20T06:40:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:17,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:17,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/add6aadfe4e44252b3257d39740cc5f0 is 50, key is test_row_0/B:col10/1732084816518/Put/seqid=0 2024-11-20T06:40:17,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741922_1098 (size=12301) 2024-11-20T06:40:17,170 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/add6aadfe4e44252b3257d39740cc5f0 2024-11-20T06:40:17,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084877175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:17,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084877181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:17,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/e792053178fe4911a2a29beeddd331ec is 50, key is test_row_0/C:col10/1732084816518/Put/seqid=0 2024-11-20T06:40:17,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741923_1099 (size=12301) 2024-11-20T06:40:17,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084877284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:17,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084877294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:17,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084877404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,407 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 8b50cf3b5c22a8ba480f30fa5ecd19ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:17,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084877413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084877413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,414 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 8b50cf3b5c22a8ba480f30fa5ecd19ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:17,415 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 8b50cf3b5c22a8ba480f30fa5ecd19ec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:17,448 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/af04226be0fc43ed92ccba7d4ec1f975 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/af04226be0fc43ed92ccba7d4ec1f975 2024-11-20T06:40:17,454 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into af04226be0fc43ed92ccba7d4ec1f975(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:17,454 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:17,454 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=13, startTime=1732084816991; duration=0sec 2024-11-20T06:40:17,454 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:17,454 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:17,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084877493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:17,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084877503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:17,600 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/e792053178fe4911a2a29beeddd331ec 2024-11-20T06:40:17,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/41d859fc1c0245f5a36bc74b71f97d2a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/41d859fc1c0245f5a36bc74b71f97d2a 2024-11-20T06:40:17,615 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/41d859fc1c0245f5a36bc74b71f97d2a, entries=150, sequenceid=405, filesize=12.0 K 2024-11-20T06:40:17,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/add6aadfe4e44252b3257d39740cc5f0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/add6aadfe4e44252b3257d39740cc5f0 2024-11-20T06:40:17,629 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/add6aadfe4e44252b3257d39740cc5f0, entries=150, sequenceid=405, filesize=12.0 K 2024-11-20T06:40:17,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/e792053178fe4911a2a29beeddd331ec as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/e792053178fe4911a2a29beeddd331ec 2024-11-20T06:40:17,640 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/e792053178fe4911a2a29beeddd331ec, entries=150, sequenceid=405, filesize=12.0 K 2024-11-20T06:40:17,642 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 530ms, sequenceid=405, compaction requested=false 2024-11-20T06:40:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:17,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-20T06:40:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-20T06:40:17,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-20T06:40:17,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9250 sec 2024-11-20T06:40:17,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.9380 sec 2024-11-20T06:40:17,815 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:17,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T06:40:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:17,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/979913523b0c45c3815b4ae248ca4a52 is 50, key is test_row_0/A:col10/1732084817157/Put/seqid=0 2024-11-20T06:40:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T06:40:17,865 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:17,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-20T06:40:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T06:40:17,869 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:17,871 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:17,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:17,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741924_1100 (size=14741) 2024-11-20T06:40:17,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/979913523b0c45c3815b4ae248ca4a52 2024-11-20T06:40:17,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/ad051f9f41e2444780be3ce281905406 is 50, key is test_row_0/B:col10/1732084817157/Put/seqid=0 2024-11-20T06:40:17,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741925_1101 (size=12301) 2024-11-20T06:40:17,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084877967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T06:40:17,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084877972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:18,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T06:40:18,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:18,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:18,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084878084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:18,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084878094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:18,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T06:40:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T06:40:18,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084878294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:18,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084878314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:18,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/ad051f9f41e2444780be3ce281905406 2024-11-20T06:40:18,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T06:40:18,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:18,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/a84e1426f87c46cab73f42673623ccd6 is 50, key is test_row_0/C:col10/1732084817157/Put/seqid=0 2024-11-20T06:40:18,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741926_1102 (size=12301) 2024-11-20T06:40:18,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T06:40:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T06:40:18,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084878603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:18,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084878624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:18,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T06:40:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:18,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/a84e1426f87c46cab73f42673623ccd6 2024-11-20T06:40:18,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/979913523b0c45c3815b4ae248ca4a52 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/979913523b0c45c3815b4ae248ca4a52 2024-11-20T06:40:18,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/979913523b0c45c3815b4ae248ca4a52, entries=200, sequenceid=419, filesize=14.4 K 2024-11-20T06:40:18,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/ad051f9f41e2444780be3ce281905406 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ad051f9f41e2444780be3ce281905406 2024-11-20T06:40:18,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ad051f9f41e2444780be3ce281905406, entries=150, sequenceid=419, filesize=12.0 K 2024-11-20T06:40:18,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/a84e1426f87c46cab73f42673623ccd6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a84e1426f87c46cab73f42673623ccd6 2024-11-20T06:40:18,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T06:40:18,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:18,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:18,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a84e1426f87c46cab73f42673623ccd6, entries=150, sequenceid=419, filesize=12.0 K 2024-11-20T06:40:18,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 989ms, sequenceid=419, compaction requested=true 2024-11-20T06:40:18,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:18,805 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:18,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:18,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:18,805 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:18,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:18,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:18,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:18,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:18,806 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:18,806 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:18,806 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,807 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/f4f5d6a6b46841b5b8baf74331ee8db1, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/add6aadfe4e44252b3257d39740cc5f0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ad051f9f41e2444780be3ce281905406] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.9 K 2024-11-20T06:40:18,807 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:18,807 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:18,807 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,807 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/af04226be0fc43ed92ccba7d4ec1f975, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/41d859fc1c0245f5a36bc74b71f97d2a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/979913523b0c45c3815b4ae248ca4a52] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=39.3 K 2024-11-20T06:40:18,807 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f5d6a6b46841b5b8baf74331ee8db1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732084815788 2024-11-20T06:40:18,807 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting af04226be0fc43ed92ccba7d4ec1f975, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732084815788 2024-11-20T06:40:18,808 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d859fc1c0245f5a36bc74b71f97d2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732084816506 2024-11-20T06:40:18,808 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting add6aadfe4e44252b3257d39740cc5f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732084816506 2024-11-20T06:40:18,809 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ad051f9f41e2444780be3ce281905406, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732084817157 2024-11-20T06:40:18,809 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 979913523b0c45c3815b4ae248ca4a52, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732084817157 2024-11-20T06:40:18,845 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:18,846 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/e868f76a5fc4431eb401c71ee21384e5 is 50, key is test_row_0/A:col10/1732084817157/Put/seqid=0 2024-11-20T06:40:18,856 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:18,857 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/c80fb41a60724cd4a4a26ba125faab38 is 50, key is test_row_0/B:col10/1732084817157/Put/seqid=0 2024-11-20T06:40:18,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741927_1103 (size=13323) 2024-11-20T06:40:18,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741928_1104 (size=13323) 2024-11-20T06:40:18,925 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/e868f76a5fc4431eb401c71ee21384e5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e868f76a5fc4431eb401c71ee21384e5 2024-11-20T06:40:18,929 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/c80fb41a60724cd4a4a26ba125faab38 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c80fb41a60724cd4a4a26ba125faab38 2024-11-20T06:40:18,943 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into e868f76a5fc4431eb401c71ee21384e5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:18,943 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:18,943 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=13, startTime=1732084818805; duration=0sec 2024-11-20T06:40:18,944 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:18,944 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:18,944 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:18,944 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into c80fb41a60724cd4a4a26ba125faab38(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:18,944 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:18,944 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=13, startTime=1732084818805; duration=0sec 2024-11-20T06:40:18,945 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:18,945 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:18,947 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:18,947 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:18,947 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,947 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/ceaff7f9a5424bfda87f8a4877a631e7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/e792053178fe4911a2a29beeddd331ec, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a84e1426f87c46cab73f42673623ccd6] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=36.9 K 2024-11-20T06:40:18,948 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceaff7f9a5424bfda87f8a4877a631e7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732084815788 2024-11-20T06:40:18,949 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting e792053178fe4911a2a29beeddd331ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732084816506 2024-11-20T06:40:18,949 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting a84e1426f87c46cab73f42673623ccd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732084817157 2024-11-20T06:40:18,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T06:40:18,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:18,953 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T06:40:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/0a09b3da70034777982e2bd86ad6a4af is 50, key is test_row_0/A:col10/1732084817964/Put/seqid=0 2024-11-20T06:40:18,979 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:18,979 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/d59e14fd777d45ada9ff58967ac0dcc4 is 50, key is test_row_0/C:col10/1732084817157/Put/seqid=0 2024-11-20T06:40:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T06:40:19,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741929_1105 (size=12301) 2024-11-20T06:40:19,020 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/0a09b3da70034777982e2bd86ad6a4af 2024-11-20T06:40:19,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741930_1106 (size=13323) 2024-11-20T06:40:19,045 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/d59e14fd777d45ada9ff58967ac0dcc4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/d59e14fd777d45ada9ff58967ac0dcc4 2024-11-20T06:40:19,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/8792488643a44a8da4b03bf99208a032 is 50, key is test_row_0/B:col10/1732084817964/Put/seqid=0 2024-11-20T06:40:19,061 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into d59e14fd777d45ada9ff58967ac0dcc4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:19,061 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:19,061 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=13, startTime=1732084818805; duration=0sec 2024-11-20T06:40:19,062 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:19,062 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:19,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741931_1107 (size=12301) 2024-11-20T06:40:19,087 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/8792488643a44a8da4b03bf99208a032 2024-11-20T06:40:19,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/51fdcb303edd48b8af9d2dd3962a71e8 is 50, key is test_row_0/C:col10/1732084817964/Put/seqid=0 2024-11-20T06:40:19,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741932_1108 (size=12301) 2024-11-20T06:40:19,113 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/51fdcb303edd48b8af9d2dd3962a71e8 2024-11-20T06:40:19,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/0a09b3da70034777982e2bd86ad6a4af as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0a09b3da70034777982e2bd86ad6a4af 2024-11-20T06:40:19,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:19,126 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0a09b3da70034777982e2bd86ad6a4af, entries=150, sequenceid=444, filesize=12.0 K 2024-11-20T06:40:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:19,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/8792488643a44a8da4b03bf99208a032 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/8792488643a44a8da4b03bf99208a032 2024-11-20T06:40:19,142 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/8792488643a44a8da4b03bf99208a032, entries=150, sequenceid=444, filesize=12.0 K 2024-11-20T06:40:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/51fdcb303edd48b8af9d2dd3962a71e8 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/51fdcb303edd48b8af9d2dd3962a71e8 2024-11-20T06:40:19,151 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/51fdcb303edd48b8af9d2dd3962a71e8, entries=150, sequenceid=444, filesize=12.0 K 2024-11-20T06:40:19,152 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=53.67 KB/54960 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 199ms, sequenceid=444, compaction requested=false 2024-11-20T06:40:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:19,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-20T06:40:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-20T06:40:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:19,155 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:19,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T06:40:19,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T06:40:19,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2830 sec 2024-11-20T06:40:19,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.2920 sec 2024-11-20T06:40:19,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/8920423a38524f5790f763f6e2464b63 is 50, key is test_row_0/A:col10/1732084819153/Put/seqid=0 2024-11-20T06:40:19,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741933_1109 (size=14741) 2024-11-20T06:40:19,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/8920423a38524f5790f763f6e2464b63 2024-11-20T06:40:19,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/0c353431d10d4338bcfaeb4f4058f0eb is 50, key is test_row_0/B:col10/1732084819153/Put/seqid=0 2024-11-20T06:40:19,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741934_1110 (size=12301) 2024-11-20T06:40:19,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084879221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084879229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084879334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084879334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45206 deadline: 1732084879424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45214 deadline: 1732084879424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,428 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 8b50cf3b5c22a8ba480f30fa5ecd19ec of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,429 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 8b50cf3b5c22a8ba480f30fa5ecd19ec of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45184 deadline: 1732084879434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,438 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 8b50cf3b5c22a8ba480f30fa5ecd19ec of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084879544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084879544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/0c353431d10d4338bcfaeb4f4058f0eb 2024-11-20T06:40:19,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/b746eb2d518b465595f4898453cf227e is 50, key is test_row_0/C:col10/1732084819153/Put/seqid=0 2024-11-20T06:40:19,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741935_1111 (size=12301) 2024-11-20T06:40:19,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/b746eb2d518b465595f4898453cf227e 2024-11-20T06:40:19,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/8920423a38524f5790f763f6e2464b63 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8920423a38524f5790f763f6e2464b63 2024-11-20T06:40:19,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8920423a38524f5790f763f6e2464b63, entries=200, sequenceid=458, filesize=14.4 K 2024-11-20T06:40:19,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/0c353431d10d4338bcfaeb4f4058f0eb as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/0c353431d10d4338bcfaeb4f4058f0eb 2024-11-20T06:40:19,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/0c353431d10d4338bcfaeb4f4058f0eb, entries=150, sequenceid=458, filesize=12.0 K 2024-11-20T06:40:19,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/b746eb2d518b465595f4898453cf227e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/b746eb2d518b465595f4898453cf227e 2024-11-20T06:40:19,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/b746eb2d518b465595f4898453cf227e, entries=150, sequenceid=458, filesize=12.0 K 2024-11-20T06:40:19,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 488ms, sequenceid=458, compaction requested=true 2024-11-20T06:40:19,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:19,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:19,643 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:19,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:19,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:19,644 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:19,645 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:19,645 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:19,645 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:19,645 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e868f76a5fc4431eb401c71ee21384e5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0a09b3da70034777982e2bd86ad6a4af, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8920423a38524f5790f763f6e2464b63] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=39.4 K 2024-11-20T06:40:19,646 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:19,646 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:19,646 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:19,646 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c80fb41a60724cd4a4a26ba125faab38, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/8792488643a44a8da4b03bf99208a032, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/0c353431d10d4338bcfaeb4f4058f0eb] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=37.0 K 2024-11-20T06:40:19,647 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting e868f76a5fc4431eb401c71ee21384e5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732084817157 2024-11-20T06:40:19,647 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a09b3da70034777982e2bd86ad6a4af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732084817906 2024-11-20T06:40:19,647 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c80fb41a60724cd4a4a26ba125faab38, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732084817157 2024-11-20T06:40:19,648 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8920423a38524f5790f763f6e2464b63, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1732084819131 2024-11-20T06:40:19,648 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8792488643a44a8da4b03bf99208a032, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732084817906 2024-11-20T06:40:19,649 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c353431d10d4338bcfaeb4f4058f0eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1732084819149 2024-11-20T06:40:19,671 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:19,671 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/9737b533b4164573a8ac749fa157c3f6 is 50, key is test_row_0/A:col10/1732084819153/Put/seqid=0 2024-11-20T06:40:19,672 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:19,672 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/bb39c7939e1f4d3987a4b7d666ac9f2b is 50, key is test_row_0/B:col10/1732084819153/Put/seqid=0 2024-11-20T06:40:19,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741936_1112 (size=13425) 2024-11-20T06:40:19,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741937_1113 (size=13425) 2024-11-20T06:40:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:19,856 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:19,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T06:40:19,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/c23fd8de068345bab0bdfe07d324e35f is 50, key is test_row_0/A:col10/1732084819210/Put/seqid=0 2024-11-20T06:40:19,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741938_1114 (size=14741) 2024-11-20T06:40:19,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/c23fd8de068345bab0bdfe07d324e35f 2024-11-20T06:40:19,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/7b1a8173c87347d5a1efbb1b732bd47f is 50, key is test_row_0/B:col10/1732084819210/Put/seqid=0 2024-11-20T06:40:19,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084879884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084879884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741939_1115 (size=12301) 2024-11-20T06:40:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:19,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084879994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084880004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T06:40:20,014 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:20,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-20T06:40:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T06:40:20,017 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:20,018 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:20,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:20,096 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/bb39c7939e1f4d3987a4b7d666ac9f2b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/bb39c7939e1f4d3987a4b7d666ac9f2b 2024-11-20T06:40:20,096 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/9737b533b4164573a8ac749fa157c3f6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/9737b533b4164573a8ac749fa157c3f6 2024-11-20T06:40:20,106 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into bb39c7939e1f4d3987a4b7d666ac9f2b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:20,106 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:20,106 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=13, startTime=1732084819644; duration=0sec 2024-11-20T06:40:20,106 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:20,106 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 9737b533b4164573a8ac749fa157c3f6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:20,107 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:20,107 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:20,107 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=13, startTime=1732084819643; duration=0sec 2024-11-20T06:40:20,107 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:20,107 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:20,107 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:20,108 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:20,108 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:20,108 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:20,108 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/d59e14fd777d45ada9ff58967ac0dcc4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/51fdcb303edd48b8af9d2dd3962a71e8, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/b746eb2d518b465595f4898453cf227e] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=37.0 K 2024-11-20T06:40:20,108 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d59e14fd777d45ada9ff58967ac0dcc4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732084817157 2024-11-20T06:40:20,109 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 51fdcb303edd48b8af9d2dd3962a71e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732084817906 2024-11-20T06:40:20,109 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b746eb2d518b465595f4898453cf227e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1732084819149 2024-11-20T06:40:20,120 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:20,121 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/8b1d4300f40e4ab7b408904b0f081db6 is 50, key is test_row_0/C:col10/1732084819153/Put/seqid=0 2024-11-20T06:40:20,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T06:40:20,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741940_1116 (size=13425) 2024-11-20T06:40:20,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T06:40:20,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:20,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:20,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:20,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:20,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:20,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084880204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:20,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:20,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084880213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:20,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/7b1a8173c87347d5a1efbb1b732bd47f 2024-11-20T06:40:20,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/4b039cfdb5bb427295ee36234d0b8740 is 50, key is test_row_0/C:col10/1732084819210/Put/seqid=0 2024-11-20T06:40:20,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T06:40:20,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:20,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:20,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:20,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:20,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T06:40:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:20,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741941_1117 (size=12301) 2024-11-20T06:40:20,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/4b039cfdb5bb427295ee36234d0b8740 2024-11-20T06:40:20,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/c23fd8de068345bab0bdfe07d324e35f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/c23fd8de068345bab0bdfe07d324e35f 2024-11-20T06:40:20,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/c23fd8de068345bab0bdfe07d324e35f, entries=200, sequenceid=482, filesize=14.4 K 2024-11-20T06:40:20,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/7b1a8173c87347d5a1efbb1b732bd47f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7b1a8173c87347d5a1efbb1b732bd47f 2024-11-20T06:40:20,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7b1a8173c87347d5a1efbb1b732bd47f, entries=150, sequenceid=482, filesize=12.0 K 2024-11-20T06:40:20,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/4b039cfdb5bb427295ee36234d0b8740 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/4b039cfdb5bb427295ee36234d0b8740 2024-11-20T06:40:20,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/4b039cfdb5bb427295ee36234d0b8740, entries=150, sequenceid=482, filesize=12.0 K 2024-11-20T06:40:20,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 513ms, sequenceid=482, compaction requested=false 2024-11-20T06:40:20,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:20,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T06:40:20,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:20,488 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T06:40:20,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/4b58b96df18a4c2ca3567f936e39188e is 50, key is test_row_0/A:col10/1732084819861/Put/seqid=0 2024-11-20T06:40:20,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741942_1118 (size=12301) 2024-11-20T06:40:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:20,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:20,537 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/8b1d4300f40e4ab7b408904b0f081db6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b1d4300f40e4ab7b408904b0f081db6 2024-11-20T06:40:20,550 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 8b1d4300f40e4ab7b408904b0f081db6(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:20,551 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:20,551 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=13, startTime=1732084819644; duration=0sec 2024-11-20T06:40:20,551 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:20,551 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:20,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084880611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:20,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084880611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T06:40:20,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:20,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084880724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084880724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:20,905 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/4b58b96df18a4c2ca3567f936e39188e 2024-11-20T06:40:20,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/25d39af4d76d41889df08676f36f3dfe is 50, key is test_row_0/B:col10/1732084819861/Put/seqid=0 2024-11-20T06:40:20,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741943_1119 (size=12301) 2024-11-20T06:40:20,930 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/25d39af4d76d41889df08676f36f3dfe 2024-11-20T06:40:20,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084880934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:20,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084880934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:20,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/903a06f6478b46b58d5b13562ad9c351 is 50, key is test_row_0/C:col10/1732084819861/Put/seqid=0 2024-11-20T06:40:20,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741944_1120 (size=12301) 2024-11-20T06:40:20,961 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/903a06f6478b46b58d5b13562ad9c351 2024-11-20T06:40:20,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/4b58b96df18a4c2ca3567f936e39188e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/4b58b96df18a4c2ca3567f936e39188e 2024-11-20T06:40:20,978 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/4b58b96df18a4c2ca3567f936e39188e, entries=150, sequenceid=496, filesize=12.0 K 2024-11-20T06:40:20,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/25d39af4d76d41889df08676f36f3dfe as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/25d39af4d76d41889df08676f36f3dfe 2024-11-20T06:40:20,987 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/25d39af4d76d41889df08676f36f3dfe, entries=150, sequenceid=496, filesize=12.0 K 2024-11-20T06:40:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/903a06f6478b46b58d5b13562ad9c351 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/903a06f6478b46b58d5b13562ad9c351 2024-11-20T06:40:20,995 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/903a06f6478b46b58d5b13562ad9c351, entries=150, sequenceid=496, filesize=12.0 K 2024-11-20T06:40:20,997 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 509ms, sequenceid=496, compaction requested=true 2024-11-20T06:40:20,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:20,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:20,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-20T06:40:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-20T06:40:21,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T06:40:21,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 985 msec 2024-11-20T06:40:21,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 993 msec 2024-11-20T06:40:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T06:40:21,154 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:21,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-20T06:40:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T06:40:21,158 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:21,160 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:21,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:21,247 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:21,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T06:40:21,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/f58b04847477401ea6911d4083a7a8cc is 50, key is test_row_0/A:col10/1732084820609/Put/seqid=0 2024-11-20T06:40:21,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741945_1121 (size=14741) 2024-11-20T06:40:21,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/f58b04847477401ea6911d4083a7a8cc 2024-11-20T06:40:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T06:40:21,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084881261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:21,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084881261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:21,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/527114585b0742e490da24c149ef5202 is 50, key is test_row_0/B:col10/1732084820609/Put/seqid=0 2024-11-20T06:40:21,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741946_1122 (size=12301) 2024-11-20T06:40:21,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T06:40:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:21,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084881374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:21,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084881374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:21,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T06:40:21,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:21,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T06:40:21,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084881584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084881584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:21,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T06:40:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:21,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,654 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-20T06:40:21,654 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:21,654 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-20T06:40:21,654 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:21,655 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-20T06:40:21,655 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:21,659 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:21,659 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:21,659 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:21,660 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:21,660 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:21,660 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:21,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:21,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:21,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:21,662 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-20T06:40:21,662 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:21,662 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:21,663 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:21,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:21,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/527114585b0742e490da24c149ef5202 2024-11-20T06:40:21,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/45b2d09e69a94e9ba6ce5719f9a00c06 is 50, key is test_row_0/C:col10/1732084820609/Put/seqid=0 2024-11-20T06:40:21,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741947_1123 (size=12301) 2024-11-20T06:40:21,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T06:40:21,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:21,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:21,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:21,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T06:40:21,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:21,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45216 deadline: 1732084881894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45156 deadline: 1732084881894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:21,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T06:40:21,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:21,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:22,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T06:40:22,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:22,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:22,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:22,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:22,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/45b2d09e69a94e9ba6ce5719f9a00c06 2024-11-20T06:40:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/f58b04847477401ea6911d4083a7a8cc as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/f58b04847477401ea6911d4083a7a8cc 2024-11-20T06:40:22,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/f58b04847477401ea6911d4083a7a8cc, entries=200, sequenceid=524, filesize=14.4 K 2024-11-20T06:40:22,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/527114585b0742e490da24c149ef5202 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/527114585b0742e490da24c149ef5202 2024-11-20T06:40:22,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/527114585b0742e490da24c149ef5202, entries=150, sequenceid=524, filesize=12.0 K 2024-11-20T06:40:22,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/45b2d09e69a94e9ba6ce5719f9a00c06 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/45b2d09e69a94e9ba6ce5719f9a00c06 2024-11-20T06:40:22,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/45b2d09e69a94e9ba6ce5719f9a00c06, entries=150, sequenceid=524, filesize=12.0 K 2024-11-20T06:40:22,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 880ms, sequenceid=524, compaction requested=true 2024-11-20T06:40:22,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:22,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:22,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:22,128 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:22,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:22,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:22,128 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:22,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b50cf3b5c22a8ba480f30fa5ecd19ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:22,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:22,129 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55208 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:22,129 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:22,129 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/A is initiating minor compaction (all files) 2024-11-20T06:40:22,129 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/B is initiating minor compaction (all files) 2024-11-20T06:40:22,129 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/A in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:22,129 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/B in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:22,129 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/9737b533b4164573a8ac749fa157c3f6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/c23fd8de068345bab0bdfe07d324e35f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/4b58b96df18a4c2ca3567f936e39188e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/f58b04847477401ea6911d4083a7a8cc] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=53.9 K 2024-11-20T06:40:22,129 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/bb39c7939e1f4d3987a4b7d666ac9f2b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7b1a8173c87347d5a1efbb1b732bd47f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/25d39af4d76d41889df08676f36f3dfe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/527114585b0742e490da24c149ef5202] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=49.1 K 2024-11-20T06:40:22,130 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9737b533b4164573a8ac749fa157c3f6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1732084819149 2024-11-20T06:40:22,130 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting bb39c7939e1f4d3987a4b7d666ac9f2b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1732084819149 2024-11-20T06:40:22,130 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting c23fd8de068345bab0bdfe07d324e35f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732084819210 2024-11-20T06:40:22,130 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b1a8173c87347d5a1efbb1b732bd47f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732084819210 2024-11-20T06:40:22,130 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b58b96df18a4c2ca3567f936e39188e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732084819861 2024-11-20T06:40:22,130 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d39af4d76d41889df08676f36f3dfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732084819861 2024-11-20T06:40:22,131 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting f58b04847477401ea6911d4083a7a8cc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1732084820609 2024-11-20T06:40:22,131 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 527114585b0742e490da24c149ef5202, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1732084820609 2024-11-20T06:40:22,139 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#A#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:22,140 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/e44bbe6b1f024b41a3606041412da8ea is 50, key is test_row_0/A:col10/1732084820609/Put/seqid=0 2024-11-20T06:40:22,140 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#B#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:22,140 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/ae5e63e290924354a72bcb8bef7b65a8 is 50, key is test_row_0/B:col10/1732084820609/Put/seqid=0 2024-11-20T06:40:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741948_1124 (size=13561) 2024-11-20T06:40:22,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741949_1125 (size=13561) 2024-11-20T06:40:22,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-20T06:40:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:22,244 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-20T06:40:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b5cd684b07d646fbbc23f3272af82166 is 50, key is test_row_0/A:col10/1732084821259/Put/seqid=0 2024-11-20T06:40:22,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741950_1126 (size=12301) 2024-11-20T06:40:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T06:40:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:22,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. as already flushing 2024-11-20T06:40:22,409 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-20T06:40:22,409 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:22,409 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:22,409 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-20T06:40:22,409 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:22,409 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:22,409 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:22,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:22,410 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:22,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:22,549 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/e44bbe6b1f024b41a3606041412da8ea as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e44bbe6b1f024b41a3606041412da8ea 2024-11-20T06:40:22,549 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/ae5e63e290924354a72bcb8bef7b65a8 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ae5e63e290924354a72bcb8bef7b65a8 2024-11-20T06:40:22,556 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/A of 8b50cf3b5c22a8ba480f30fa5ecd19ec into e44bbe6b1f024b41a3606041412da8ea(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:22,556 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/B of 8b50cf3b5c22a8ba480f30fa5ecd19ec into ae5e63e290924354a72bcb8bef7b65a8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:22,556 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:22,556 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:22,556 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/B, priority=12, startTime=1732084822128; duration=0sec 2024-11-20T06:40:22,556 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/A, priority=12, startTime=1732084822128; duration=0sec 2024-11-20T06:40:22,556 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:22,556 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:B 2024-11-20T06:40:22,556 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:22,556 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:A 2024-11-20T06:40:22,556 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:22,558 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:22,558 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 8b50cf3b5c22a8ba480f30fa5ecd19ec/C is initiating minor compaction (all files) 2024-11-20T06:40:22,558 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8b50cf3b5c22a8ba480f30fa5ecd19ec/C in TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:22,559 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b1d4300f40e4ab7b408904b0f081db6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/4b039cfdb5bb427295ee36234d0b8740, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/903a06f6478b46b58d5b13562ad9c351, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/45b2d09e69a94e9ba6ce5719f9a00c06] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp, totalSize=49.1 K 2024-11-20T06:40:22,559 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b1d4300f40e4ab7b408904b0f081db6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1732084819149 2024-11-20T06:40:22,560 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b039cfdb5bb427295ee36234d0b8740, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1732084819210 2024-11-20T06:40:22,560 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 903a06f6478b46b58d5b13562ad9c351, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1732084819861 2024-11-20T06:40:22,560 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b2d09e69a94e9ba6ce5719f9a00c06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1732084820609 2024-11-20T06:40:22,571 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b50cf3b5c22a8ba480f30fa5ecd19ec#C#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:22,571 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/505d796d806649888fee289215d5bc93 is 50, key is test_row_0/C:col10/1732084820609/Put/seqid=0 2024-11-20T06:40:22,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741951_1127 (size=13561) 2024-11-20T06:40:22,658 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b5cd684b07d646fbbc23f3272af82166 2024-11-20T06:40:22,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/687a9efc2dda49878dc64de50f5f3bd6 is 50, key is test_row_0/B:col10/1732084821259/Put/seqid=0 2024-11-20T06:40:22,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741952_1128 (size=12301) 2024-11-20T06:40:22,878 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T06:40:23,024 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/505d796d806649888fee289215d5bc93 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/505d796d806649888fee289215d5bc93 2024-11-20T06:40:23,034 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8b50cf3b5c22a8ba480f30fa5ecd19ec/C of 8b50cf3b5c22a8ba480f30fa5ecd19ec into 505d796d806649888fee289215d5bc93(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:23,034 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:23,034 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., storeName=8b50cf3b5c22a8ba480f30fa5ecd19ec/C, priority=12, startTime=1732084822128; duration=0sec 2024-11-20T06:40:23,034 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:23,034 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b50cf3b5c22a8ba480f30fa5ecd19ec:C 2024-11-20T06:40:23,073 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/687a9efc2dda49878dc64de50f5f3bd6 2024-11-20T06:40:23,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/cf3f74acdb5c431ba49a9a8b92e7bd0f is 50, key is test_row_0/C:col10/1732084821259/Put/seqid=0 2024-11-20T06:40:23,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741953_1129 (size=12301) 2024-11-20T06:40:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T06:40:23,461 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-20T06:40:23,461 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-20T06:40:23,461 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:23,461 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:23,461 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:23,461 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:23,461 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:23,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:23,462 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:23,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:23,466 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-20T06:40:23,466 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:23,466 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:23,466 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:23,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:23,485 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/cf3f74acdb5c431ba49a9a8b92e7bd0f 2024-11-20T06:40:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/b5cd684b07d646fbbc23f3272af82166 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b5cd684b07d646fbbc23f3272af82166 2024-11-20T06:40:23,501 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b5cd684b07d646fbbc23f3272af82166, entries=150, sequenceid=533, filesize=12.0 K 2024-11-20T06:40:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/687a9efc2dda49878dc64de50f5f3bd6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/687a9efc2dda49878dc64de50f5f3bd6 2024-11-20T06:40:23,506 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/687a9efc2dda49878dc64de50f5f3bd6, entries=150, sequenceid=533, filesize=12.0 K 2024-11-20T06:40:23,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/cf3f74acdb5c431ba49a9a8b92e7bd0f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/cf3f74acdb5c431ba49a9a8b92e7bd0f 2024-11-20T06:40:23,512 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/cf3f74acdb5c431ba49a9a8b92e7bd0f, entries=150, sequenceid=533, filesize=12.0 K 2024-11-20T06:40:23,513 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=33.54 KB/34350 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 1270ms, sequenceid=533, compaction requested=false 2024-11-20T06:40:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: 2024-11-20T06:40:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-20T06:40:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-20T06:40:23,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-20T06:40:23,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3540 sec 2024-11-20T06:40:23,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 2.3610 sec 2024-11-20T06:40:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T06:40:25,315 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:25,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T06:40:25,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-20T06:40:25,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-20T06:40:25,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-20T06:40:25,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-20T06:40:25,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-20T06:40:25,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T06:40:25,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3096 2024-11-20T06:40:25,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3032 2024-11-20T06:40:25,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T06:40:25,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1443 2024-11-20T06:40:25,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4324 rows 2024-11-20T06:40:25,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1454 2024-11-20T06:40:25,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4357 rows 2024-11-20T06:40:25,316 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:40:25,316 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:40:25,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:25,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:25,317 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:25,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T06:40:25,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T06:40:25,331 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084825331"}]},"ts":"1732084825331"} 2024-11-20T06:40:25,334 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T06:40:25,334 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T06:40:25,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T06:40:25,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b50cf3b5c22a8ba480f30fa5ecd19ec, UNASSIGN}] 2024-11-20T06:40:25,343 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b50cf3b5c22a8ba480f30fa5ecd19ec, UNASSIGN 2024-11-20T06:40:25,345 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=8b50cf3b5c22a8ba480f30fa5ecd19ec, regionState=CLOSING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:25,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b50cf3b5c22a8ba480f30fa5ecd19ec, UNASSIGN because future has completed 2024-11-20T06:40:25,348 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T06:40:25,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:40:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T06:40:25,506 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:25,506 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T06:40:25,507 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 8b50cf3b5c22a8ba480f30fa5ecd19ec, disabling compactions & flushes 2024-11-20T06:40:25,507 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:25,507 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:25,507 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. after waiting 0 ms 2024-11-20T06:40:25,507 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:25,507 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T06:40:25,513 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/33877cffa10345b489f80be2c3de147d is 50, key is test_row_1/A:col10/1732084823464/Put/seqid=0 2024-11-20T06:40:25,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741954_1130 (size=9857) 2024-11-20T06:40:25,529 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/33877cffa10345b489f80be2c3de147d 2024-11-20T06:40:25,544 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/39b5df00e0cb4626be04bac98029d28f is 50, key is test_row_1/B:col10/1732084823464/Put/seqid=0 2024-11-20T06:40:25,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741955_1131 (size=9857) 2024-11-20T06:40:25,555 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/39b5df00e0cb4626be04bac98029d28f 2024-11-20T06:40:25,566 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/9ee62c55c2f04938986f6494c50d80bb is 50, key is test_row_1/C:col10/1732084823464/Put/seqid=0 2024-11-20T06:40:25,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741956_1132 (size=9857) 2024-11-20T06:40:25,580 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/9ee62c55c2f04938986f6494c50d80bb 2024-11-20T06:40:25,588 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/A/33877cffa10345b489f80be2c3de147d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/33877cffa10345b489f80be2c3de147d 2024-11-20T06:40:25,595 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/33877cffa10345b489f80be2c3de147d, entries=100, sequenceid=544, filesize=9.6 K 2024-11-20T06:40:25,597 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/B/39b5df00e0cb4626be04bac98029d28f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/39b5df00e0cb4626be04bac98029d28f 2024-11-20T06:40:25,605 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/39b5df00e0cb4626be04bac98029d28f, entries=100, sequenceid=544, filesize=9.6 K 2024-11-20T06:40:25,607 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/.tmp/C/9ee62c55c2f04938986f6494c50d80bb as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/9ee62c55c2f04938986f6494c50d80bb 2024-11-20T06:40:25,615 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/9ee62c55c2f04938986f6494c50d80bb, entries=100, sequenceid=544, filesize=9.6 K 2024-11-20T06:40:25,616 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 109ms, sequenceid=544, compaction requested=true 2024-11-20T06:40:25,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/259c19a3cea64e83871dfed22ee3dbf3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/3a06d0e3deac413ba3303a75122eac0f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/19020a929339461f86aac585aae39eeb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/630d9877382a4171a54fae730e15edd2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b6abf57731f6466e81a274c29d143d52, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1c49477c8e4449c9b257d251ae2b87bc, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/d54918dce2724f87af256449b1ebd226, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/05215c616f4f4e8bb9a5da0f2517c88e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/5d5b6be1d75b49189840b18d26249838, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e2e4a61d7c544c7f87b8e6d4cea1628c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/358c9ffaea384cc7866a2f277c8b9faa, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b127fc8d72434fc4abeb8cf7e0238277, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/66db5f3d3e914b3b9b57602ad72491a0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/03f771d6e8d1492f9c8358a7a5c8157c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b65c54c6727b41c98fa2fa07084b0ae8, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/01973a32b6ad4db99d03d3aa02fca40a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b14d4fbf72dd4f52bcef007d6ec82505, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/98539334b7e94b5c82374997e004bb3e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/ea7161d0609146ca94381e77f4fe44ae, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/656b476ad6ce44a58922df859d48d2db, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1acde39aff7a461380f2250dec6c410e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/2aa0a5bbeec448ccbea52cfa49e7f566, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8bbcccb014ee432892116a59c78adb68, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/926e558898d3463b9582ca16c66c7c41, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/648ae321fb9c44ffbf906c9f69c22b51, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/849b9d396c2a488e92e072d175d19436, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/af04226be0fc43ed92ccba7d4ec1f975, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0664e77badde441faf735f20eafcdcc7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/41d859fc1c0245f5a36bc74b71f97d2a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/979913523b0c45c3815b4ae248ca4a52, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e868f76a5fc4431eb401c71ee21384e5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0a09b3da70034777982e2bd86ad6a4af, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8920423a38524f5790f763f6e2464b63, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/9737b533b4164573a8ac749fa157c3f6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/c23fd8de068345bab0bdfe07d324e35f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/4b58b96df18a4c2ca3567f936e39188e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/f58b04847477401ea6911d4083a7a8cc] to archive 2024-11-20T06:40:25,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:40:25,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/259c19a3cea64e83871dfed22ee3dbf3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/259c19a3cea64e83871dfed22ee3dbf3 2024-11-20T06:40:25,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/3a06d0e3deac413ba3303a75122eac0f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/3a06d0e3deac413ba3303a75122eac0f 2024-11-20T06:40:25,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/19020a929339461f86aac585aae39eeb to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/19020a929339461f86aac585aae39eeb 2024-11-20T06:40:25,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/630d9877382a4171a54fae730e15edd2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/630d9877382a4171a54fae730e15edd2 2024-11-20T06:40:25,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b6abf57731f6466e81a274c29d143d52 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b6abf57731f6466e81a274c29d143d52 2024-11-20T06:40:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T06:40:25,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1c49477c8e4449c9b257d251ae2b87bc to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1c49477c8e4449c9b257d251ae2b87bc 2024-11-20T06:40:25,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/d54918dce2724f87af256449b1ebd226 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/d54918dce2724f87af256449b1ebd226 2024-11-20T06:40:25,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/05215c616f4f4e8bb9a5da0f2517c88e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/05215c616f4f4e8bb9a5da0f2517c88e 2024-11-20T06:40:25,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/5d5b6be1d75b49189840b18d26249838 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/5d5b6be1d75b49189840b18d26249838 2024-11-20T06:40:25,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e2e4a61d7c544c7f87b8e6d4cea1628c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e2e4a61d7c544c7f87b8e6d4cea1628c 2024-11-20T06:40:25,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/358c9ffaea384cc7866a2f277c8b9faa to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/358c9ffaea384cc7866a2f277c8b9faa 2024-11-20T06:40:25,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b127fc8d72434fc4abeb8cf7e0238277 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b127fc8d72434fc4abeb8cf7e0238277 2024-11-20T06:40:25,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/66db5f3d3e914b3b9b57602ad72491a0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/66db5f3d3e914b3b9b57602ad72491a0 2024-11-20T06:40:25,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/03f771d6e8d1492f9c8358a7a5c8157c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/03f771d6e8d1492f9c8358a7a5c8157c 2024-11-20T06:40:25,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b65c54c6727b41c98fa2fa07084b0ae8 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b65c54c6727b41c98fa2fa07084b0ae8 2024-11-20T06:40:25,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/01973a32b6ad4db99d03d3aa02fca40a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/01973a32b6ad4db99d03d3aa02fca40a 2024-11-20T06:40:25,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b14d4fbf72dd4f52bcef007d6ec82505 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b14d4fbf72dd4f52bcef007d6ec82505 2024-11-20T06:40:25,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/98539334b7e94b5c82374997e004bb3e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/98539334b7e94b5c82374997e004bb3e 2024-11-20T06:40:25,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/ea7161d0609146ca94381e77f4fe44ae to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/ea7161d0609146ca94381e77f4fe44ae 2024-11-20T06:40:25,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/656b476ad6ce44a58922df859d48d2db to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/656b476ad6ce44a58922df859d48d2db 2024-11-20T06:40:25,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1acde39aff7a461380f2250dec6c410e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/1acde39aff7a461380f2250dec6c410e 2024-11-20T06:40:25,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/2aa0a5bbeec448ccbea52cfa49e7f566 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/2aa0a5bbeec448ccbea52cfa49e7f566 2024-11-20T06:40:25,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8bbcccb014ee432892116a59c78adb68 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8bbcccb014ee432892116a59c78adb68 2024-11-20T06:40:25,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/926e558898d3463b9582ca16c66c7c41 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/926e558898d3463b9582ca16c66c7c41 2024-11-20T06:40:25,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/648ae321fb9c44ffbf906c9f69c22b51 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/648ae321fb9c44ffbf906c9f69c22b51 2024-11-20T06:40:25,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/849b9d396c2a488e92e072d175d19436 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/849b9d396c2a488e92e072d175d19436 2024-11-20T06:40:25,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/af04226be0fc43ed92ccba7d4ec1f975 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/af04226be0fc43ed92ccba7d4ec1f975 2024-11-20T06:40:25,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0664e77badde441faf735f20eafcdcc7 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0664e77badde441faf735f20eafcdcc7 2024-11-20T06:40:25,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/41d859fc1c0245f5a36bc74b71f97d2a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/41d859fc1c0245f5a36bc74b71f97d2a 2024-11-20T06:40:25,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/979913523b0c45c3815b4ae248ca4a52 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/979913523b0c45c3815b4ae248ca4a52 2024-11-20T06:40:25,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e868f76a5fc4431eb401c71ee21384e5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e868f76a5fc4431eb401c71ee21384e5 2024-11-20T06:40:25,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0a09b3da70034777982e2bd86ad6a4af to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/0a09b3da70034777982e2bd86ad6a4af 2024-11-20T06:40:25,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8920423a38524f5790f763f6e2464b63 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/8920423a38524f5790f763f6e2464b63 2024-11-20T06:40:25,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/9737b533b4164573a8ac749fa157c3f6 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/9737b533b4164573a8ac749fa157c3f6 2024-11-20T06:40:25,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/c23fd8de068345bab0bdfe07d324e35f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/c23fd8de068345bab0bdfe07d324e35f 2024-11-20T06:40:25,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/4b58b96df18a4c2ca3567f936e39188e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/4b58b96df18a4c2ca3567f936e39188e 2024-11-20T06:40:25,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/f58b04847477401ea6911d4083a7a8cc to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/f58b04847477401ea6911d4083a7a8cc 2024-11-20T06:40:25,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/93f8480ec219402e8e52ae569a5800e4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/89e319f70a354bb3b4a6d719f45d6f5f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/77ba247a45cc4d418118311fdee6a57a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/08eb030ec17247bb84c1fa4bb94e349f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7a06c40fe8d441ecb1ae6ac53905abf3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4efef04d3ec64700a36796476662bbfe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e5c93e7da6d34b04b541167cfc9bf21a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/df3d54bc08bf48e88c4b88bb5e0c5bd7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c3be69b1548443aa92d700a0e39df1a2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e242f1b8a712494e8437d5791e6fda21, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/759ed9fba0284a4c968046edd7accf23, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/a09dc4b6ed3343e2a7ef1506c0af7fd3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/9c7f93b5296e4703afaf496d514ccaae, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/107841891c4240e2af3be5bb26221915, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/b783744ed81040c09ad88991f717155b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/88d39d43acb640a7899895165c672fea, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/6dc099192b5540d589a7a5a715da382d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/21d1b46b964943849a8d54c267c130b9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ff69632a0f8f4832a4892b32b0f39864, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/dcda523c30044384a057e53911555842, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/afc443fdc68e42958cabdd3e1ac72a4b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/128a8d545045409f95c7fcd508179510, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/fd3ecf6d045649f69af75ba7be756f26, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4afb7d7136874ffc920c1e2d3eaaca22, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/42625d0e37024817b34f671b9ca57318, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/569271cb1cc94c818e3bf0e37fa1c076, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/f4f5d6a6b46841b5b8baf74331ee8db1, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/2f7968b63e7946108b99d9a3bb9a8c1e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/add6aadfe4e44252b3257d39740cc5f0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c80fb41a60724cd4a4a26ba125faab38, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ad051f9f41e2444780be3ce281905406, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/8792488643a44a8da4b03bf99208a032, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/bb39c7939e1f4d3987a4b7d666ac9f2b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/0c353431d10d4338bcfaeb4f4058f0eb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7b1a8173c87347d5a1efbb1b732bd47f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/25d39af4d76d41889df08676f36f3dfe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/527114585b0742e490da24c149ef5202] to archive 2024-11-20T06:40:25,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:40:25,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/93f8480ec219402e8e52ae569a5800e4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/93f8480ec219402e8e52ae569a5800e4 2024-11-20T06:40:25,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/89e319f70a354bb3b4a6d719f45d6f5f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/89e319f70a354bb3b4a6d719f45d6f5f 2024-11-20T06:40:25,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/77ba247a45cc4d418118311fdee6a57a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/77ba247a45cc4d418118311fdee6a57a 2024-11-20T06:40:25,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/08eb030ec17247bb84c1fa4bb94e349f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/08eb030ec17247bb84c1fa4bb94e349f 2024-11-20T06:40:25,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7a06c40fe8d441ecb1ae6ac53905abf3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7a06c40fe8d441ecb1ae6ac53905abf3 2024-11-20T06:40:25,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4efef04d3ec64700a36796476662bbfe to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4efef04d3ec64700a36796476662bbfe 2024-11-20T06:40:25,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e5c93e7da6d34b04b541167cfc9bf21a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e5c93e7da6d34b04b541167cfc9bf21a 2024-11-20T06:40:25,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/df3d54bc08bf48e88c4b88bb5e0c5bd7 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/df3d54bc08bf48e88c4b88bb5e0c5bd7 2024-11-20T06:40:25,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c3be69b1548443aa92d700a0e39df1a2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c3be69b1548443aa92d700a0e39df1a2 2024-11-20T06:40:25,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e242f1b8a712494e8437d5791e6fda21 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/e242f1b8a712494e8437d5791e6fda21 2024-11-20T06:40:25,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/759ed9fba0284a4c968046edd7accf23 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/759ed9fba0284a4c968046edd7accf23 2024-11-20T06:40:25,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/a09dc4b6ed3343e2a7ef1506c0af7fd3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/a09dc4b6ed3343e2a7ef1506c0af7fd3 2024-11-20T06:40:25,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/9c7f93b5296e4703afaf496d514ccaae to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/9c7f93b5296e4703afaf496d514ccaae 2024-11-20T06:40:25,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/107841891c4240e2af3be5bb26221915 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/107841891c4240e2af3be5bb26221915 2024-11-20T06:40:25,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/b783744ed81040c09ad88991f717155b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/b783744ed81040c09ad88991f717155b 2024-11-20T06:40:25,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/88d39d43acb640a7899895165c672fea to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/88d39d43acb640a7899895165c672fea 2024-11-20T06:40:25,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/6dc099192b5540d589a7a5a715da382d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/6dc099192b5540d589a7a5a715da382d 2024-11-20T06:40:25,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/21d1b46b964943849a8d54c267c130b9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/21d1b46b964943849a8d54c267c130b9 2024-11-20T06:40:25,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ff69632a0f8f4832a4892b32b0f39864 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ff69632a0f8f4832a4892b32b0f39864 2024-11-20T06:40:25,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/dcda523c30044384a057e53911555842 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/dcda523c30044384a057e53911555842 2024-11-20T06:40:25,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/afc443fdc68e42958cabdd3e1ac72a4b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/afc443fdc68e42958cabdd3e1ac72a4b 2024-11-20T06:40:25,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/128a8d545045409f95c7fcd508179510 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/128a8d545045409f95c7fcd508179510 2024-11-20T06:40:25,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/fd3ecf6d045649f69af75ba7be756f26 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/fd3ecf6d045649f69af75ba7be756f26 2024-11-20T06:40:25,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4afb7d7136874ffc920c1e2d3eaaca22 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/4afb7d7136874ffc920c1e2d3eaaca22 2024-11-20T06:40:25,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/42625d0e37024817b34f671b9ca57318 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/42625d0e37024817b34f671b9ca57318 2024-11-20T06:40:25,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/569271cb1cc94c818e3bf0e37fa1c076 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/569271cb1cc94c818e3bf0e37fa1c076 2024-11-20T06:40:25,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/f4f5d6a6b46841b5b8baf74331ee8db1 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/f4f5d6a6b46841b5b8baf74331ee8db1 2024-11-20T06:40:25,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/2f7968b63e7946108b99d9a3bb9a8c1e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/2f7968b63e7946108b99d9a3bb9a8c1e 2024-11-20T06:40:25,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/add6aadfe4e44252b3257d39740cc5f0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/add6aadfe4e44252b3257d39740cc5f0 2024-11-20T06:40:25,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c80fb41a60724cd4a4a26ba125faab38 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/c80fb41a60724cd4a4a26ba125faab38 2024-11-20T06:40:25,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ad051f9f41e2444780be3ce281905406 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ad051f9f41e2444780be3ce281905406 2024-11-20T06:40:25,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/8792488643a44a8da4b03bf99208a032 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/8792488643a44a8da4b03bf99208a032 2024-11-20T06:40:25,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/bb39c7939e1f4d3987a4b7d666ac9f2b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/bb39c7939e1f4d3987a4b7d666ac9f2b 2024-11-20T06:40:25,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/0c353431d10d4338bcfaeb4f4058f0eb to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/0c353431d10d4338bcfaeb4f4058f0eb 2024-11-20T06:40:25,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7b1a8173c87347d5a1efbb1b732bd47f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/7b1a8173c87347d5a1efbb1b732bd47f 2024-11-20T06:40:25,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/25d39af4d76d41889df08676f36f3dfe to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/25d39af4d76d41889df08676f36f3dfe 2024-11-20T06:40:25,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/527114585b0742e490da24c149ef5202 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/527114585b0742e490da24c149ef5202 2024-11-20T06:40:25,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/03e230f7bfec40319d10277e8ce34f57, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/fb28761e24424eb69352109b70d76ed2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/c649333dfe7f4ba08b42fee158186666, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/591886a93388441cb5c9b11cfddc4387, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/83e0afc73945441e9c077e4f4c2d471f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/3cfd2910d45348eb86025c5ce752a3d5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/db4f78a822f44c8cac1d4084a0f8aef0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/dd898e992e8746ec9ffacb020e4ab0fb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/377415db16c949b4b1344a151c63b88c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7255d4ecd50d4bbf9893dfb33a437815, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a0739a9ab41944baa22e0d4c1aba452e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b53c76cd62040a88820bc10d350df2f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/863388e0e9c244adaf85f751ebff3ab0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/ce06e771defa4c8ea3b41cef0938852b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/59c6bf18e2be4187867b251bbee97972, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/be5b80a573ff4625af64073442218c9a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/44046fedceef4ddda333a41bc502cda2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/aff9d5bc5dbd4b5dab396c9e3095e614, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/bda2a4ef36b5416295d74da56ce5f10b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7e83dabd83b84bc69e7caf82a8ebfe31, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0fa5dbebf7174160947d43b7152fb742, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/325e33bfec354cd881fd99075370929f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/9427ceb96f964ac6a04ca1a2ca969e47, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/473e126ae93d4b12a92fe89772065ce4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6d505f61b36d48da9bcad7b3e5082a6a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6fa72b0c0efe47c9903aaf46dcf85ece, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/ceaff7f9a5424bfda87f8a4877a631e7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0c97c6bfa35b42869d99e8ec47d0f910, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/e792053178fe4911a2a29beeddd331ec, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/d59e14fd777d45ada9ff58967ac0dcc4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a84e1426f87c46cab73f42673623ccd6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/51fdcb303edd48b8af9d2dd3962a71e8, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b1d4300f40e4ab7b408904b0f081db6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/b746eb2d518b465595f4898453cf227e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/4b039cfdb5bb427295ee36234d0b8740, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/903a06f6478b46b58d5b13562ad9c351, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/45b2d09e69a94e9ba6ce5719f9a00c06] to archive 2024-11-20T06:40:25,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:40:25,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/03e230f7bfec40319d10277e8ce34f57 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/03e230f7bfec40319d10277e8ce34f57 2024-11-20T06:40:25,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/fb28761e24424eb69352109b70d76ed2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/fb28761e24424eb69352109b70d76ed2 2024-11-20T06:40:25,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/c649333dfe7f4ba08b42fee158186666 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/c649333dfe7f4ba08b42fee158186666 2024-11-20T06:40:25,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/591886a93388441cb5c9b11cfddc4387 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/591886a93388441cb5c9b11cfddc4387 2024-11-20T06:40:25,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/83e0afc73945441e9c077e4f4c2d471f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/83e0afc73945441e9c077e4f4c2d471f 2024-11-20T06:40:25,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/3cfd2910d45348eb86025c5ce752a3d5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/3cfd2910d45348eb86025c5ce752a3d5 2024-11-20T06:40:25,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/db4f78a822f44c8cac1d4084a0f8aef0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/db4f78a822f44c8cac1d4084a0f8aef0 2024-11-20T06:40:25,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/dd898e992e8746ec9ffacb020e4ab0fb to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/dd898e992e8746ec9ffacb020e4ab0fb 2024-11-20T06:40:25,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/377415db16c949b4b1344a151c63b88c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/377415db16c949b4b1344a151c63b88c 2024-11-20T06:40:25,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7255d4ecd50d4bbf9893dfb33a437815 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7255d4ecd50d4bbf9893dfb33a437815 2024-11-20T06:40:25,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a0739a9ab41944baa22e0d4c1aba452e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a0739a9ab41944baa22e0d4c1aba452e 2024-11-20T06:40:25,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b53c76cd62040a88820bc10d350df2f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b53c76cd62040a88820bc10d350df2f 2024-11-20T06:40:25,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/863388e0e9c244adaf85f751ebff3ab0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/863388e0e9c244adaf85f751ebff3ab0 2024-11-20T06:40:25,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/ce06e771defa4c8ea3b41cef0938852b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/ce06e771defa4c8ea3b41cef0938852b 2024-11-20T06:40:25,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/59c6bf18e2be4187867b251bbee97972 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/59c6bf18e2be4187867b251bbee97972 2024-11-20T06:40:25,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/be5b80a573ff4625af64073442218c9a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/be5b80a573ff4625af64073442218c9a 2024-11-20T06:40:25,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/44046fedceef4ddda333a41bc502cda2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/44046fedceef4ddda333a41bc502cda2 2024-11-20T06:40:25,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/aff9d5bc5dbd4b5dab396c9e3095e614 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/aff9d5bc5dbd4b5dab396c9e3095e614 2024-11-20T06:40:25,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/bda2a4ef36b5416295d74da56ce5f10b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/bda2a4ef36b5416295d74da56ce5f10b 2024-11-20T06:40:25,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7e83dabd83b84bc69e7caf82a8ebfe31 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/7e83dabd83b84bc69e7caf82a8ebfe31 2024-11-20T06:40:25,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0fa5dbebf7174160947d43b7152fb742 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0fa5dbebf7174160947d43b7152fb742 2024-11-20T06:40:25,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/325e33bfec354cd881fd99075370929f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/325e33bfec354cd881fd99075370929f 2024-11-20T06:40:25,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/9427ceb96f964ac6a04ca1a2ca969e47 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/9427ceb96f964ac6a04ca1a2ca969e47 2024-11-20T06:40:25,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/473e126ae93d4b12a92fe89772065ce4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/473e126ae93d4b12a92fe89772065ce4 2024-11-20T06:40:25,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6d505f61b36d48da9bcad7b3e5082a6a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6d505f61b36d48da9bcad7b3e5082a6a 2024-11-20T06:40:25,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6fa72b0c0efe47c9903aaf46dcf85ece to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/6fa72b0c0efe47c9903aaf46dcf85ece 2024-11-20T06:40:25,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/ceaff7f9a5424bfda87f8a4877a631e7 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/ceaff7f9a5424bfda87f8a4877a631e7 2024-11-20T06:40:25,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0c97c6bfa35b42869d99e8ec47d0f910 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/0c97c6bfa35b42869d99e8ec47d0f910 2024-11-20T06:40:25,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/e792053178fe4911a2a29beeddd331ec to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/e792053178fe4911a2a29beeddd331ec 2024-11-20T06:40:25,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/d59e14fd777d45ada9ff58967ac0dcc4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/d59e14fd777d45ada9ff58967ac0dcc4 2024-11-20T06:40:25,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a84e1426f87c46cab73f42673623ccd6 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/a84e1426f87c46cab73f42673623ccd6 2024-11-20T06:40:25,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/51fdcb303edd48b8af9d2dd3962a71e8 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/51fdcb303edd48b8af9d2dd3962a71e8 2024-11-20T06:40:25,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b1d4300f40e4ab7b408904b0f081db6 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/8b1d4300f40e4ab7b408904b0f081db6 2024-11-20T06:40:25,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/b746eb2d518b465595f4898453cf227e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/b746eb2d518b465595f4898453cf227e 2024-11-20T06:40:25,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/4b039cfdb5bb427295ee36234d0b8740 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/4b039cfdb5bb427295ee36234d0b8740 2024-11-20T06:40:25,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/903a06f6478b46b58d5b13562ad9c351 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/903a06f6478b46b58d5b13562ad9c351 2024-11-20T06:40:25,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/45b2d09e69a94e9ba6ce5719f9a00c06 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/45b2d09e69a94e9ba6ce5719f9a00c06 2024-11-20T06:40:25,850 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/recovered.edits/547.seqid, newMaxSeqId=547, maxSeqId=1 2024-11-20T06:40:25,850 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. 2024-11-20T06:40:25,851 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 8b50cf3b5c22a8ba480f30fa5ecd19ec: Waiting for close lock at 1732084825507Running coprocessor pre-close hooks at 1732084825507Disabling compacts and flushes for region at 1732084825507Disabling writes for close at 1732084825507Obtaining lock to block concurrent updates at 1732084825507Preparing flush snapshotting stores in 8b50cf3b5c22a8ba480f30fa5ecd19ec at 1732084825507Finished memstore snapshotting TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732084825508 (+1 ms)Flushing stores of TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec. at 1732084825509 (+1 ms)Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec/A: creating writer at 1732084825509Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec/A: appending metadata at 1732084825512 (+3 ms)Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec/A: closing flushed file at 1732084825512Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec/B: creating writer at 1732084825539 (+27 ms)Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec/B: appending metadata at 1732084825544 (+5 ms)Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec/B: closing flushed file at 1732084825544Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec/C: creating writer at 1732084825561 (+17 ms)Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec/C: appending metadata at 1732084825565 (+4 ms)Flushing 8b50cf3b5c22a8ba480f30fa5ecd19ec/C: closing flushed file at 1732084825565Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11ab838b: reopening flushed file at 1732084825587 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51ad18ab: reopening flushed file at 1732084825596 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34d397cc: reopening flushed file at 1732084825605 (+9 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8b50cf3b5c22a8ba480f30fa5ecd19ec in 109ms, sequenceid=544, compaction requested=true at 1732084825616 (+11 ms)Writing region close event to WAL at 1732084825846 (+230 ms)Running coprocessor post-close hooks at 1732084825850 (+4 ms)Closed at 1732084825850 2024-11-20T06:40:25,853 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:25,854 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=8b50cf3b5c22a8ba480f30fa5ecd19ec, regionState=CLOSED 2024-11-20T06:40:25,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:40:25,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-20T06:40:25,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 8b50cf3b5c22a8ba480f30fa5ecd19ec, server=61c2066a8a0f,33847,1732084794617 in 511 msec 2024-11-20T06:40:25,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-20T06:40:25,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b50cf3b5c22a8ba480f30fa5ecd19ec, UNASSIGN in 520 msec 2024-11-20T06:40:25,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-20T06:40:25,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-20T06:40:25,867 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084825867"}]},"ts":"1732084825867"} 2024-11-20T06:40:25,869 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T06:40:25,869 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T06:40:25,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 546 msec 2024-11-20T06:40:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T06:40:25,955 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:25,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T06:40:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:25,965 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:25,967 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-20T06:40:25,972 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:25,977 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/recovered.edits] 2024-11-20T06:40:25,981 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/33877cffa10345b489f80be2c3de147d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/33877cffa10345b489f80be2c3de147d 2024-11-20T06:40:25,983 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b5cd684b07d646fbbc23f3272af82166 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/b5cd684b07d646fbbc23f3272af82166 2024-11-20T06:40:25,984 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e44bbe6b1f024b41a3606041412da8ea to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/A/e44bbe6b1f024b41a3606041412da8ea 2024-11-20T06:40:25,987 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/39b5df00e0cb4626be04bac98029d28f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/39b5df00e0cb4626be04bac98029d28f 2024-11-20T06:40:25,988 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/687a9efc2dda49878dc64de50f5f3bd6 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/687a9efc2dda49878dc64de50f5f3bd6 2024-11-20T06:40:25,989 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ae5e63e290924354a72bcb8bef7b65a8 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/B/ae5e63e290924354a72bcb8bef7b65a8 2024-11-20T06:40:25,992 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/505d796d806649888fee289215d5bc93 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/505d796d806649888fee289215d5bc93 2024-11-20T06:40:25,993 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/9ee62c55c2f04938986f6494c50d80bb to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/9ee62c55c2f04938986f6494c50d80bb 2024-11-20T06:40:25,995 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/cf3f74acdb5c431ba49a9a8b92e7bd0f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/C/cf3f74acdb5c431ba49a9a8b92e7bd0f 2024-11-20T06:40:25,998 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/recovered.edits/547.seqid to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec/recovered.edits/547.seqid 2024-11-20T06:40:25,999 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/8b50cf3b5c22a8ba480f30fa5ecd19ec 2024-11-20T06:40:25,999 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T06:40:26,005 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:26,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-20T06:40:26,016 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T06:40:26,053 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T06:40:26,055 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:26,055 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T06:40:26,055 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732084826055"}]},"ts":"9223372036854775807"} 2024-11-20T06:40:26,061 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T06:40:26,061 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8b50cf3b5c22a8ba480f30fa5ecd19ec, NAME => 'TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T06:40:26,062 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T06:40:26,062 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732084826062"}]},"ts":"9223372036854775807"} 2024-11-20T06:40:26,067 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T06:40:26,069 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:26,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 110 msec 2024-11-20T06:40:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-20T06:40:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T06:40:26,081 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:26,097 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1351162969_22 at /127.0.0.1:53162 [Waiting for operation #372] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1532015227_22 at /127.0.0.1:48380 [Waiting for operation #376] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;61c2066a8a0f:33847-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=459 (was 323) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5125 (was 5165) 2024-11-20T06:40:26,109 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=459, ProcessCount=11, AvailableMemoryMB=5125 2024-11-20T06:40:26,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T06:40:26,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:40:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:26,115 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T06:40:26,115 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:26,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-20T06:40:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T06:40:26,116 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T06:40:26,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741957_1133 (size=959) 2024-11-20T06:40:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T06:40:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T06:40:26,533 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5 2024-11-20T06:40:26,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741958_1134 (size=53) 2024-11-20T06:40:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T06:40:26,943 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:40:26,943 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d735f3ed3d64a5ca97cafea9e7136d03, disabling compactions & flushes 2024-11-20T06:40:26,943 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:26,943 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:26,943 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. after waiting 0 ms 2024-11-20T06:40:26,943 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:26,943 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:26,943 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d735f3ed3d64a5ca97cafea9e7136d03: Waiting for close lock at 1732084826943Disabling compacts and flushes for region at 1732084826943Disabling writes for close at 1732084826943Writing region close event to WAL at 1732084826943Closed at 1732084826943 2024-11-20T06:40:26,945 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T06:40:26,945 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732084826945"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732084826945"}]},"ts":"1732084826945"} 2024-11-20T06:40:26,947 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T06:40:26,948 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T06:40:26,948 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084826948"}]},"ts":"1732084826948"} 2024-11-20T06:40:26,950 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T06:40:26,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, ASSIGN}] 2024-11-20T06:40:26,952 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, ASSIGN 2024-11-20T06:40:26,953 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, ASSIGN; state=OFFLINE, location=61c2066a8a0f,33847,1732084794617; forceNewPlan=false, retain=false 2024-11-20T06:40:27,104 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=d735f3ed3d64a5ca97cafea9e7136d03, regionState=OPENING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:27,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, ASSIGN because future has completed 2024-11-20T06:40:27,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:40:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T06:40:27,262 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:27,262 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:40:27,262 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,262 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:40:27,262 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,262 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,264 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,266 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d735f3ed3d64a5ca97cafea9e7136d03 columnFamilyName A 2024-11-20T06:40:27,266 DEBUG [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:27,267 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(327): Store=d735f3ed3d64a5ca97cafea9e7136d03/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:27,267 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,269 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d735f3ed3d64a5ca97cafea9e7136d03 columnFamilyName B 2024-11-20T06:40:27,269 DEBUG [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:27,269 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(327): Store=d735f3ed3d64a5ca97cafea9e7136d03/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:27,269 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,271 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d735f3ed3d64a5ca97cafea9e7136d03 columnFamilyName C 2024-11-20T06:40:27,271 DEBUG [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:27,272 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(327): Store=d735f3ed3d64a5ca97cafea9e7136d03/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:27,272 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,273 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,273 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,274 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,274 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,274 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:40:27,275 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,277 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:40:27,278 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened d735f3ed3d64a5ca97cafea9e7136d03; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64181446, jitterRate=-0.04362192749977112}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:40:27,278 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:27,278 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for d735f3ed3d64a5ca97cafea9e7136d03: Running coprocessor pre-open hook at 1732084827262Writing region info on filesystem at 1732084827262Initializing all the Stores at 1732084827263 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084827263Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084827264 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084827264Cleaning up temporary data from old regions at 1732084827274 (+10 ms)Running coprocessor post-open hooks at 1732084827278 (+4 ms)Region opened successfully at 1732084827278 2024-11-20T06:40:27,279 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., pid=36, masterSystemTime=1732084827259 2024-11-20T06:40:27,281 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:27,281 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:27,282 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=d735f3ed3d64a5ca97cafea9e7136d03, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:27,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:40:27,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-20T06:40:27,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 in 178 msec 2024-11-20T06:40:27,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-20T06:40:27,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, ASSIGN in 336 msec 2024-11-20T06:40:27,289 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T06:40:27,289 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084827289"}]},"ts":"1732084827289"} 2024-11-20T06:40:27,291 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T06:40:27,292 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T06:40:27,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-20T06:40:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-20T06:40:28,264 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:28,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:28,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:28,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:28,267 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c846e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:28,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:28,269 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30dfdb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:28,270 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:28,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:28,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:28,274 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T06:40:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,33659,1732084793752 2024-11-20T06:40:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@75dbc7e6 2024-11-20T06:40:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T06:40:28,276 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T06:40:28,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T06:40:28,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:40:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:28,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741959_1135 (size=995) 2024-11-20T06:40:28,702 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T06:40:28,702 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T06:40:28,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T06:40:28,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, REOPEN/MOVE}] 2024-11-20T06:40:28,715 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, REOPEN/MOVE 2024-11-20T06:40:28,716 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=d735f3ed3d64a5ca97cafea9e7136d03, regionState=CLOSING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:28,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, REOPEN/MOVE because future has completed 2024-11-20T06:40:28,718 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T06:40:28,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:40:28,871 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:28,871 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T06:40:28,871 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing d735f3ed3d64a5ca97cafea9e7136d03, disabling compactions & flushes 2024-11-20T06:40:28,871 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:28,871 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:28,871 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. after waiting 0 ms 2024-11-20T06:40:28,871 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:28,875 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T06:40:28,875 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:28,875 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for d735f3ed3d64a5ca97cafea9e7136d03: Waiting for close lock at 1732084828871Running coprocessor pre-close hooks at 1732084828871Disabling compacts and flushes for region at 1732084828871Disabling writes for close at 1732084828871Writing region close event to WAL at 1732084828871Running coprocessor post-close hooks at 1732084828875 (+4 ms)Closed at 1732084828875 2024-11-20T06:40:28,875 WARN [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: d735f3ed3d64a5ca97cafea9e7136d03 to self. 2024-11-20T06:40:28,877 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:28,878 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=d735f3ed3d64a5ca97cafea9e7136d03, regionState=CLOSED 2024-11-20T06:40:28,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:40:28,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-20T06:40:28,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 in 162 msec 2024-11-20T06:40:28,882 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, REOPEN/MOVE; state=CLOSED, location=61c2066a8a0f,33847,1732084794617; forceNewPlan=false, retain=true 2024-11-20T06:40:29,034 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=d735f3ed3d64a5ca97cafea9e7136d03, regionState=OPENING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, REOPEN/MOVE because future has completed 2024-11-20T06:40:29,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:40:29,194 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,194 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:40:29,195 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,195 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:40:29,195 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,195 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,200 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,208 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d735f3ed3d64a5ca97cafea9e7136d03 columnFamilyName A 2024-11-20T06:40:29,210 DEBUG [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:29,211 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(327): Store=d735f3ed3d64a5ca97cafea9e7136d03/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:29,212 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,213 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d735f3ed3d64a5ca97cafea9e7136d03 columnFamilyName B 2024-11-20T06:40:29,213 DEBUG [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:29,214 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(327): Store=d735f3ed3d64a5ca97cafea9e7136d03/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:29,214 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,215 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d735f3ed3d64a5ca97cafea9e7136d03 columnFamilyName C 2024-11-20T06:40:29,215 DEBUG [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:29,215 INFO [StoreOpener-d735f3ed3d64a5ca97cafea9e7136d03-1 {}] regionserver.HStore(327): Store=d735f3ed3d64a5ca97cafea9e7136d03/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:29,216 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,216 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,217 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,218 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,218 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,219 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:40:29,220 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,221 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened d735f3ed3d64a5ca97cafea9e7136d03; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70468299, jitterRate=0.0500594824552536}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:40:29,221 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,223 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for d735f3ed3d64a5ca97cafea9e7136d03: Running coprocessor pre-open hook at 1732084829195Writing region info on filesystem at 1732084829195Initializing all the Stores at 1732084829196 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084829196Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084829197 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084829197Cleaning up temporary data from old regions at 1732084829218 (+21 ms)Running coprocessor post-open hooks at 1732084829221 (+3 ms)Region opened successfully at 1732084829222 (+1 ms) 2024-11-20T06:40:29,223 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., pid=41, masterSystemTime=1732084829190 2024-11-20T06:40:29,226 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,226 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,226 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=d735f3ed3d64a5ca97cafea9e7136d03, regionState=OPEN, openSeqNum=5, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:40:29,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-20T06:40:29,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 in 191 msec 2024-11-20T06:40:29,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-20T06:40:29,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, REOPEN/MOVE in 516 msec 2024-11-20T06:40:29,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-20T06:40:29,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 526 msec 2024-11-20T06:40:29,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 953 msec 2024-11-20T06:40:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-20T06:40:29,242 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:29,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13460079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:29,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:29,244 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d64178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:29,246 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:29,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d1de7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a297d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:29,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:29,248 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1358217d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:29,250 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3981e3d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bc0752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:29,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:29,253 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e0a15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:29,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:29,255 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:29,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9f5d50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c85e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:29,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:29,258 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f151f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:29,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:29,260 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c367d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ab7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,262 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:29,262 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:29,264 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3860cc67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:29,266 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c116249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1987d108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:29,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:29,269 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45935292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:29,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:29,271 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4069ceef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5fd39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:29,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:29,276 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c300ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:29,277 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18847951, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18690ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:29,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:29,282 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4662add3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:29,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:29,285 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6c65e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e58620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:29,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:29,296 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244a9fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:29,298 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9f9fc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:29,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:29,300 DEBUG [Thread-681 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:29,300 DEBUG [Thread-678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:29,300 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:29,301 DEBUG [Thread-687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:29,301 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:29,301 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:29,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-20T06:40:29,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T06:40:29,308 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:29,309 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:29,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:29,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:29,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:29,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:29,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:40:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:40:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:40:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:40:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:40:29,323 DEBUG [Thread-676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:29,324 DEBUG [Thread-685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:29,325 DEBUG [Thread-683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:29,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:40:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:29,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:29,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:40:29,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:40:29,342 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:29,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T06:40:29,342 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:29,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:29,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:40:29,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203e4536415c384b649d3a48db752a8bf8_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084829332/Put/seqid=0 2024-11-20T06:40:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084889388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084889394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084889395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084889396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084889398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T06:40:29,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741960_1136 (size=14594) 2024-11-20T06:40:29,423 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:29,430 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203e4536415c384b649d3a48db752a8bf8_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203e4536415c384b649d3a48db752a8bf8_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:29,433 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/375d74f5ef20485eb48b4041bd6139fe, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:29,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/375d74f5ef20485eb48b4041bd6139fe is 175, key is test_row_0/A:col10/1732084829332/Put/seqid=0 2024-11-20T06:40:29,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T06:40:29,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:29,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741961_1137 (size=39549) 2024-11-20T06:40:29,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084889504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084889517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084889525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084889526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084889526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T06:40:29,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T06:40:29,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:29,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084889715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084889724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084889744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084889744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:29,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084889744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:29,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:29,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:29,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T06:40:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,889 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/375d74f5ef20485eb48b4041bd6139fe 2024-11-20T06:40:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T06:40:29,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T06:40:29,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:29,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:29,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/dbb73f56e3a64557888a4292728748f5 is 50, key is test_row_0/B:col10/1732084829332/Put/seqid=0 2024-11-20T06:40:29,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741962_1138 (size=12001) 2024-11-20T06:40:29,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/dbb73f56e3a64557888a4292728748f5 2024-11-20T06:40:30,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/e467d068e689449db9f6670187f431f3 is 50, key is test_row_0/C:col10/1732084829332/Put/seqid=0 2024-11-20T06:40:30,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084890034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084890044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741963_1139 (size=12001) 2024-11-20T06:40:30,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/e467d068e689449db9f6670187f431f3 2024-11-20T06:40:30,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/375d74f5ef20485eb48b4041bd6139fe as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/375d74f5ef20485eb48b4041bd6139fe 2024-11-20T06:40:30,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084890064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084890064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084890064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/375d74f5ef20485eb48b4041bd6139fe, entries=200, sequenceid=17, filesize=38.6 K 2024-11-20T06:40:30,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/dbb73f56e3a64557888a4292728748f5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/dbb73f56e3a64557888a4292728748f5 2024-11-20T06:40:30,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T06:40:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:30,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:30,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/dbb73f56e3a64557888a4292728748f5, entries=150, sequenceid=17, filesize=11.7 K 2024-11-20T06:40:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:30,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/e467d068e689449db9f6670187f431f3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/e467d068e689449db9f6670187f431f3 2024-11-20T06:40:30,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/e467d068e689449db9f6670187f431f3, entries=150, sequenceid=17, filesize=11.7 K 2024-11-20T06:40:30,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d735f3ed3d64a5ca97cafea9e7136d03 in 761ms, sequenceid=17, compaction requested=false 2024-11-20T06:40:30,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:30,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T06:40:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:30,247 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T06:40:30,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205a43b49e12214a718248a6a0c4cdbf99_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084829384/Put/seqid=0 2024-11-20T06:40:30,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741964_1140 (size=12154) 2024-11-20T06:40:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T06:40:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:30,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084890595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084890597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084890603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084890604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084890606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:30,714 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205a43b49e12214a718248a6a0c4cdbf99_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205a43b49e12214a718248a6a0c4cdbf99_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:30,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/2da9714fc0e74548b561dc5136f827c5, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:30,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/2da9714fc0e74548b561dc5136f827c5 is 175, key is test_row_0/A:col10/1732084829384/Put/seqid=0 2024-11-20T06:40:30,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084890715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084890715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084890715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084890724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084890724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741965_1141 (size=30955) 2024-11-20T06:40:30,756 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/2da9714fc0e74548b561dc5136f827c5 2024-11-20T06:40:30,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/2d41fa7aa5c54d95b6f9982f83f4123c is 50, key is test_row_0/B:col10/1732084829384/Put/seqid=0 2024-11-20T06:40:30,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741966_1142 (size=12001) 2024-11-20T06:40:30,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084890934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084890934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084890934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084890934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:30,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084890934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,208 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/2d41fa7aa5c54d95b6f9982f83f4123c 2024-11-20T06:40:31,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/1392e10989d04132b075ba54f7733559 is 50, key is test_row_0/C:col10/1732084829384/Put/seqid=0 2024-11-20T06:40:31,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741967_1143 (size=12001) 2024-11-20T06:40:31,232 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/1392e10989d04132b075ba54f7733559 2024-11-20T06:40:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/2da9714fc0e74548b561dc5136f827c5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/2da9714fc0e74548b561dc5136f827c5 2024-11-20T06:40:31,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084891246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084891246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084891246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084891251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084891252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,259 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/2da9714fc0e74548b561dc5136f827c5, entries=150, sequenceid=40, filesize=30.2 K 2024-11-20T06:40:31,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/2d41fa7aa5c54d95b6f9982f83f4123c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/2d41fa7aa5c54d95b6f9982f83f4123c 2024-11-20T06:40:31,270 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/2d41fa7aa5c54d95b6f9982f83f4123c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T06:40:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/1392e10989d04132b075ba54f7733559 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/1392e10989d04132b075ba54f7733559 2024-11-20T06:40:31,277 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/1392e10989d04132b075ba54f7733559, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T06:40:31,278 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d735f3ed3d64a5ca97cafea9e7136d03 in 1031ms, sequenceid=40, compaction requested=false 2024-11-20T06:40:31,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:31,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:31,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-20T06:40:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-20T06:40:31,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-20T06:40:31,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9710 sec 2024-11-20T06:40:31,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9840 sec 2024-11-20T06:40:31,397 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T06:40:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T06:40:31,455 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:31,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-20T06:40:31,461 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T06:40:31,462 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:31,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T06:40:31,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T06:40:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:31,617 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T06:40:31,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f83cdaa44e9145d785d24b179ad32f30_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084830603/Put/seqid=0 2024-11-20T06:40:31,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741968_1144 (size=12154) 2024-11-20T06:40:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:31,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T06:40:31,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084891794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084891794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084891800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084891800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084891802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084891914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084891914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084891915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084891915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:31,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084891923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:31,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:32,058 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f83cdaa44e9145d785d24b179ad32f30_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f83cdaa44e9145d785d24b179ad32f30_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:32,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/658e197a1fed4c40b8c2b3848bfed465, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:32,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/658e197a1fed4c40b8c2b3848bfed465 is 175, key is test_row_0/A:col10/1732084830603/Put/seqid=0 2024-11-20T06:40:32,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741969_1145 (size=30955) 2024-11-20T06:40:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T06:40:32,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084892124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084892125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084892125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084892125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084892134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084892433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084892444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084892444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084892444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084892444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,471 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/658e197a1fed4c40b8c2b3848bfed465 2024-11-20T06:40:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/cf222933bb4647f6a7e872632b753d3f is 50, key is test_row_0/B:col10/1732084830603/Put/seqid=0 2024-11-20T06:40:32,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741970_1146 (size=12001) 2024-11-20T06:40:32,502 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/cf222933bb4647f6a7e872632b753d3f 2024-11-20T06:40:32,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/f216cee46a5d420e9e38e654920057f9 is 50, key is test_row_0/C:col10/1732084830603/Put/seqid=0 2024-11-20T06:40:32,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741971_1147 (size=12001) 2024-11-20T06:40:32,541 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/f216cee46a5d420e9e38e654920057f9 2024-11-20T06:40:32,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/658e197a1fed4c40b8c2b3848bfed465 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/658e197a1fed4c40b8c2b3848bfed465 2024-11-20T06:40:32,560 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/658e197a1fed4c40b8c2b3848bfed465, entries=150, sequenceid=53, filesize=30.2 K 2024-11-20T06:40:32,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/cf222933bb4647f6a7e872632b753d3f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/cf222933bb4647f6a7e872632b753d3f 2024-11-20T06:40:32,567 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/cf222933bb4647f6a7e872632b753d3f, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T06:40:32,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/f216cee46a5d420e9e38e654920057f9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/f216cee46a5d420e9e38e654920057f9 2024-11-20T06:40:32,576 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/f216cee46a5d420e9e38e654920057f9, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T06:40:32,577 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d735f3ed3d64a5ca97cafea9e7136d03 in 961ms, sequenceid=53, compaction requested=true 2024-11-20T06:40:32,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:32,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:32,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-20T06:40:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-20T06:40:32,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-20T06:40:32,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1170 sec 2024-11-20T06:40:32,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.1240 sec 2024-11-20T06:40:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T06:40:32,596 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:32,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-20T06:40:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T06:40:32,602 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:32,604 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:32,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T06:40:32,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T06:40:32,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:32,757 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T06:40:32,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206668c9429f344344ab8ae283004943e8_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084831797/Put/seqid=0 2024-11-20T06:40:32,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741972_1148 (size=12154) 2024-11-20T06:40:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T06:40:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:32,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:32,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084892965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084892967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084892970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084892971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084892972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:32,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084893074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084893074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084893094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084893094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084893094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:33,188 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206668c9429f344344ab8ae283004943e8_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206668c9429f344344ab8ae283004943e8_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:33,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/3c4a52d940434939841ccc3da9fc0026, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:33,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/3c4a52d940434939841ccc3da9fc0026 is 175, key is test_row_0/A:col10/1732084831797/Put/seqid=0 2024-11-20T06:40:33,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741973_1149 (size=30955) 2024-11-20T06:40:33,200 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/3c4a52d940434939841ccc3da9fc0026 2024-11-20T06:40:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/119061bca67c4976aadc9c861e02f6f0 is 50, key is test_row_0/B:col10/1732084831797/Put/seqid=0 2024-11-20T06:40:33,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741974_1150 (size=12001) 2024-11-20T06:40:33,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T06:40:33,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084893284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084893284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084893303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084893303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084893304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084893593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084893593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084893614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084893614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084893614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:33,620 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/119061bca67c4976aadc9c861e02f6f0 2024-11-20T06:40:33,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/9cb1c65787554f5f8898bb3d2e0f69b1 is 50, key is test_row_0/C:col10/1732084831797/Put/seqid=0 2024-11-20T06:40:33,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741975_1151 (size=12001) 2024-11-20T06:40:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T06:40:34,053 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/9cb1c65787554f5f8898bb3d2e0f69b1 2024-11-20T06:40:34,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/3c4a52d940434939841ccc3da9fc0026 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/3c4a52d940434939841ccc3da9fc0026 2024-11-20T06:40:34,067 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/3c4a52d940434939841ccc3da9fc0026, entries=150, sequenceid=76, filesize=30.2 K 2024-11-20T06:40:34,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/119061bca67c4976aadc9c861e02f6f0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/119061bca67c4976aadc9c861e02f6f0 2024-11-20T06:40:34,083 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/119061bca67c4976aadc9c861e02f6f0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-20T06:40:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/9cb1c65787554f5f8898bb3d2e0f69b1 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/9cb1c65787554f5f8898bb3d2e0f69b1 2024-11-20T06:40:34,093 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/9cb1c65787554f5f8898bb3d2e0f69b1, entries=150, sequenceid=76, filesize=11.7 K 2024-11-20T06:40:34,094 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d735f3ed3d64a5ca97cafea9e7136d03 in 1337ms, sequenceid=76, compaction requested=true 2024-11-20T06:40:34,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:34,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:34,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-20T06:40:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-20T06:40:34,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-20T06:40:34,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4920 sec 2024-11-20T06:40:34,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.5020 sec 2024-11-20T06:40:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:34,109 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:34,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T06:40:34,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112025b04a8a12cf42638ce09d0d7ad43632_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084834108/Put/seqid=0 2024-11-20T06:40:34,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741976_1152 (size=17034) 2024-11-20T06:40:34,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084894138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084894140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084894141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084894143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084894145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084894253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084894253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084894253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084894254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084894254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084894464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084894464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084894464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084894464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084894464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,536 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:34,541 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112025b04a8a12cf42638ce09d0d7ad43632_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112025b04a8a12cf42638ce09d0d7ad43632_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:34,542 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/a2776a1801074014a599ad0bdbc27b94, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:34,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/a2776a1801074014a599ad0bdbc27b94 is 175, key is test_row_0/A:col10/1732084834108/Put/seqid=0 2024-11-20T06:40:34,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741977_1153 (size=48139) 2024-11-20T06:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T06:40:34,744 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:34,745 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-20T06:40:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T06:40:34,747 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:34,748 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:34,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:34,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084894774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084894774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084894774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084894774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084894774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:34,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T06:40:34,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T06:40:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:34,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:34,952 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/a2776a1801074014a599ad0bdbc27b94 2024-11-20T06:40:34,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/c55ec5c8df4c443ca3eacab491b63aca is 50, key is test_row_0/B:col10/1732084834108/Put/seqid=0 2024-11-20T06:40:35,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741978_1154 (size=12001) 2024-11-20T06:40:35,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/c55ec5c8df4c443ca3eacab491b63aca 2024-11-20T06:40:35,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T06:40:35,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:35,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:35,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:35,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:35,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T06:40:35,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/ab9cb831977947cdaa15b511734a0557 is 50, key is test_row_0/C:col10/1732084834108/Put/seqid=0 2024-11-20T06:40:35,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741979_1155 (size=12001) 2024-11-20T06:40:35,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/ab9cb831977947cdaa15b511734a0557 2024-11-20T06:40:35,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/a2776a1801074014a599ad0bdbc27b94 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/a2776a1801074014a599ad0bdbc27b94 2024-11-20T06:40:35,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/a2776a1801074014a599ad0bdbc27b94, entries=250, sequenceid=92, filesize=47.0 K 2024-11-20T06:40:35,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/c55ec5c8df4c443ca3eacab491b63aca as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/c55ec5c8df4c443ca3eacab491b63aca 2024-11-20T06:40:35,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/c55ec5c8df4c443ca3eacab491b63aca, entries=150, sequenceid=92, filesize=11.7 K 2024-11-20T06:40:35,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/ab9cb831977947cdaa15b511734a0557 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ab9cb831977947cdaa15b511734a0557 2024-11-20T06:40:35,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ab9cb831977947cdaa15b511734a0557, entries=150, sequenceid=92, filesize=11.7 K 2024-11-20T06:40:35,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d735f3ed3d64a5ca97cafea9e7136d03 in 1013ms, sequenceid=92, compaction requested=true 2024-11-20T06:40:35,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:35,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:35,122 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T06:40:35,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:35,122 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T06:40:35,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:35,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:35,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:35,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:35,125 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T06:40:35,125 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/B is initiating minor compaction (all files) 2024-11-20T06:40:35,125 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/B in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:35,125 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/dbb73f56e3a64557888a4292728748f5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/2d41fa7aa5c54d95b6f9982f83f4123c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/cf222933bb4647f6a7e872632b753d3f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/119061bca67c4976aadc9c861e02f6f0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/c55ec5c8df4c443ca3eacab491b63aca] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=58.6 K 2024-11-20T06:40:35,125 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 180553 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T06:40:35,126 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/A is initiating minor compaction (all files) 2024-11-20T06:40:35,126 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/A in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:35,126 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/375d74f5ef20485eb48b4041bd6139fe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/2da9714fc0e74548b561dc5136f827c5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/658e197a1fed4c40b8c2b3848bfed465, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/3c4a52d940434939841ccc3da9fc0026, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/a2776a1801074014a599ad0bdbc27b94] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=176.3 K 2024-11-20T06:40:35,126 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:35,126 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/375d74f5ef20485eb48b4041bd6139fe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/2da9714fc0e74548b561dc5136f827c5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/658e197a1fed4c40b8c2b3848bfed465, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/3c4a52d940434939841ccc3da9fc0026, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/a2776a1801074014a599ad0bdbc27b94] 2024-11-20T06:40:35,127 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting dbb73f56e3a64557888a4292728748f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732084829332 2024-11-20T06:40:35,127 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d41fa7aa5c54d95b6f9982f83f4123c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732084829384 2024-11-20T06:40:35,127 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 375d74f5ef20485eb48b4041bd6139fe, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732084829326 2024-11-20T06:40:35,128 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting cf222933bb4647f6a7e872632b753d3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732084830584 2024-11-20T06:40:35,128 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2da9714fc0e74548b561dc5136f827c5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732084829384 2024-11-20T06:40:35,128 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 119061bca67c4976aadc9c861e02f6f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732084831797 2024-11-20T06:40:35,128 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 658e197a1fed4c40b8c2b3848bfed465, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732084830584 2024-11-20T06:40:35,129 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c55ec5c8df4c443ca3eacab491b63aca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732084832963 2024-11-20T06:40:35,129 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c4a52d940434939841ccc3da9fc0026, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732084831797 2024-11-20T06:40:35,129 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2776a1801074014a599ad0bdbc27b94, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732084832963 2024-11-20T06:40:35,149 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:35,149 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/df565b904a62496d9c0895937a75d51f is 50, key is test_row_0/B:col10/1732084834108/Put/seqid=0 2024-11-20T06:40:35,151 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:35,173 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411202b699ecadb894e459348532a4a5f3970_d735f3ed3d64a5ca97cafea9e7136d03 store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:35,186 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411202b699ecadb894e459348532a4a5f3970_d735f3ed3d64a5ca97cafea9e7136d03, store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:35,187 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202b699ecadb894e459348532a4a5f3970_d735f3ed3d64a5ca97cafea9e7136d03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:35,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741980_1156 (size=12173) 2024-11-20T06:40:35,209 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/df565b904a62496d9c0895937a75d51f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/df565b904a62496d9c0895937a75d51f 2024-11-20T06:40:35,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T06:40:35,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:35,211 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:40:35,223 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/B of d735f3ed3d64a5ca97cafea9e7136d03 into df565b904a62496d9c0895937a75d51f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:35,223 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:35,223 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/B, priority=11, startTime=1732084835122; duration=0sec 2024-11-20T06:40:35,223 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:35,223 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:B 2024-11-20T06:40:35,223 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T06:40:35,228 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T06:40:35,228 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/C is initiating minor compaction (all files) 2024-11-20T06:40:35,228 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/C in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:35,229 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/e467d068e689449db9f6670187f431f3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/1392e10989d04132b075ba54f7733559, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/f216cee46a5d420e9e38e654920057f9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/9cb1c65787554f5f8898bb3d2e0f69b1, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ab9cb831977947cdaa15b511734a0557] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=58.6 K 2024-11-20T06:40:35,229 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting e467d068e689449db9f6670187f431f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732084829332 2024-11-20T06:40:35,229 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 1392e10989d04132b075ba54f7733559, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732084829384 2024-11-20T06:40:35,231 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting f216cee46a5d420e9e38e654920057f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732084830584 2024-11-20T06:40:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120727fa60674cc4b8c99ca0bcd1713bf71_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084834144/Put/seqid=0 2024-11-20T06:40:35,236 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb1c65787554f5f8898bb3d2e0f69b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732084831797 2024-11-20T06:40:35,237 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ab9cb831977947cdaa15b511734a0557, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732084832963 2024-11-20T06:40:35,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741981_1157 (size=4469) 2024-11-20T06:40:35,260 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#C#compaction#138 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:35,261 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/074830ea7c474949acda70998efaaca4 is 50, key is test_row_0/C:col10/1732084834108/Put/seqid=0 2024-11-20T06:40:35,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741982_1158 (size=12154) 2024-11-20T06:40:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:35,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:35,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:35,293 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120727fa60674cc4b8c99ca0bcd1713bf71_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120727fa60674cc4b8c99ca0bcd1713bf71_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/45c4ca627c0b4cd8b0b3579eef684bb2, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/45c4ca627c0b4cd8b0b3579eef684bb2 is 175, key is test_row_0/A:col10/1732084834144/Put/seqid=0 2024-11-20T06:40:35,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084895302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084895301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084895303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741983_1159 (size=12173) 2024-11-20T06:40:35,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084895311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084895313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,336 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/074830ea7c474949acda70998efaaca4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/074830ea7c474949acda70998efaaca4 2024-11-20T06:40:35,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741984_1160 (size=30955) 2024-11-20T06:40:35,341 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/45c4ca627c0b4cd8b0b3579eef684bb2 2024-11-20T06:40:35,345 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/C of d735f3ed3d64a5ca97cafea9e7136d03 into 074830ea7c474949acda70998efaaca4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:35,346 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:35,346 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/C, priority=11, startTime=1732084835122; duration=0sec 2024-11-20T06:40:35,346 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:35,346 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:C 2024-11-20T06:40:35,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/b22af5f28e144eefae793ff4236bf785 is 50, key is test_row_0/B:col10/1732084834144/Put/seqid=0 2024-11-20T06:40:35,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741985_1161 (size=12001) 2024-11-20T06:40:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T06:40:35,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084895414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084895424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084895425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084895425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084895434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084895624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084895634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084895635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,644 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#A#compaction#136 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:35,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084895644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,645 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/c926402bf75e4da8a04211015ce2d0de is 175, key is test_row_0/A:col10/1732084834108/Put/seqid=0 2024-11-20T06:40:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084895647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741986_1162 (size=31127) 2024-11-20T06:40:35,660 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/c926402bf75e4da8a04211015ce2d0de as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/c926402bf75e4da8a04211015ce2d0de 2024-11-20T06:40:35,669 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/A of d735f3ed3d64a5ca97cafea9e7136d03 into c926402bf75e4da8a04211015ce2d0de(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:35,669 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:35,670 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/A, priority=11, startTime=1732084835122; duration=0sec 2024-11-20T06:40:35,670 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:35,670 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:A 2024-11-20T06:40:35,769 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/b22af5f28e144eefae793ff4236bf785 2024-11-20T06:40:35,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/940d5aee2e124a10afe665308ef4322a is 50, key is test_row_0/C:col10/1732084834144/Put/seqid=0 2024-11-20T06:40:35,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741987_1163 (size=12001) 2024-11-20T06:40:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T06:40:35,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084895944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,945 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084895954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084895954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084895954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084895954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:35,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,186 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/940d5aee2e124a10afe665308ef4322a 2024-11-20T06:40:36,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/45c4ca627c0b4cd8b0b3579eef684bb2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/45c4ca627c0b4cd8b0b3579eef684bb2 2024-11-20T06:40:36,208 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/45c4ca627c0b4cd8b0b3579eef684bb2, entries=150, sequenceid=112, filesize=30.2 K 2024-11-20T06:40:36,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/b22af5f28e144eefae793ff4236bf785 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b22af5f28e144eefae793ff4236bf785 2024-11-20T06:40:36,219 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b22af5f28e144eefae793ff4236bf785, entries=150, sequenceid=112, filesize=11.7 K 2024-11-20T06:40:36,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/940d5aee2e124a10afe665308ef4322a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/940d5aee2e124a10afe665308ef4322a 2024-11-20T06:40:36,226 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/940d5aee2e124a10afe665308ef4322a, entries=150, sequenceid=112, filesize=11.7 K 2024-11-20T06:40:36,227 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d735f3ed3d64a5ca97cafea9e7136d03 in 1016ms, sequenceid=112, compaction requested=false 2024-11-20T06:40:36,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:36,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:36,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-20T06:40:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-20T06:40:36,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-20T06:40:36,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4820 sec 2024-11-20T06:40:36,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4880 sec 2024-11-20T06:40:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:36,456 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:36,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T06:40:36,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112015adfd8c0ca6417189f0a60ec5aac8e0_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084835301/Put/seqid=0 2024-11-20T06:40:36,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741988_1164 (size=14644) 2024-11-20T06:40:36,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084896488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084896488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084896489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084896490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084896495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084896604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084896604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084896604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084896604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084896615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084896814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084896814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084896814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084896814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084896823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:36,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:36,877 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:36,883 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112015adfd8c0ca6417189f0a60ec5aac8e0_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112015adfd8c0ca6417189f0a60ec5aac8e0_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:36,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/75b7dfce836541b1901f25dfe4043a15, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:36,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/75b7dfce836541b1901f25dfe4043a15 is 175, key is test_row_0/A:col10/1732084835301/Put/seqid=0 2024-11-20T06:40:36,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741989_1165 (size=39599) 2024-11-20T06:40:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T06:40:36,895 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:36,895 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-20T06:40:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T06:40:36,898 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:36,899 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:36,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T06:40:37,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T06:40:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:37,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084897124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:37,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084897124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:37,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084897124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:37,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:37,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:37,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:37,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084897124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:37,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084897134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:37,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T06:40:37,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:37,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T06:40:37,289 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/75b7dfce836541b1901f25dfe4043a15 2024-11-20T06:40:37,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/111f36f47da847a3beb55e80f7da61b4 is 50, key is test_row_0/B:col10/1732084835301/Put/seqid=0 2024-11-20T06:40:37,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741990_1166 (size=12051) 2024-11-20T06:40:37,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T06:40:37,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:37,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T06:40:37,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:37,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T06:40:37,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084897634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084897634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:37,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084897634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:37,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084897644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:37,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084897644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:37,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T06:40:37,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:37,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/111f36f47da847a3beb55e80f7da61b4 2024-11-20T06:40:37,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/d161b34b91fc47d2bfff1b517b977233 is 50, key is test_row_0/C:col10/1732084835301/Put/seqid=0 2024-11-20T06:40:37,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741991_1167 (size=12051) 2024-11-20T06:40:37,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T06:40:37,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:37,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T06:40:37,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:37,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:37,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T06:40:38,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T06:40:38,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:38,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:38,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:38,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:38,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:38,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/d161b34b91fc47d2bfff1b517b977233 2024-11-20T06:40:38,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/75b7dfce836541b1901f25dfe4043a15 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/75b7dfce836541b1901f25dfe4043a15 2024-11-20T06:40:38,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/75b7dfce836541b1901f25dfe4043a15, entries=200, sequenceid=132, filesize=38.7 K 2024-11-20T06:40:38,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/111f36f47da847a3beb55e80f7da61b4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/111f36f47da847a3beb55e80f7da61b4 2024-11-20T06:40:38,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/111f36f47da847a3beb55e80f7da61b4, entries=150, sequenceid=132, filesize=11.8 K 2024-11-20T06:40:38,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/d161b34b91fc47d2bfff1b517b977233 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/d161b34b91fc47d2bfff1b517b977233 2024-11-20T06:40:38,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/d161b34b91fc47d2bfff1b517b977233, entries=150, sequenceid=132, filesize=11.8 K 2024-11-20T06:40:38,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for d735f3ed3d64a5ca97cafea9e7136d03 in 1728ms, sequenceid=132, compaction requested=true 2024-11-20T06:40:38,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:38,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:38,184 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:38,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:38,184 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:38,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:38,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:38,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:38,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:38,186 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101681 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:38,186 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:38,186 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/B is initiating minor compaction (all files) 2024-11-20T06:40:38,186 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/A is initiating minor compaction (all files) 2024-11-20T06:40:38,187 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/B in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:38,187 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/df565b904a62496d9c0895937a75d51f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b22af5f28e144eefae793ff4236bf785, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/111f36f47da847a3beb55e80f7da61b4] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=35.4 K 2024-11-20T06:40:38,187 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/A in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:38,187 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/c926402bf75e4da8a04211015ce2d0de, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/45c4ca627c0b4cd8b0b3579eef684bb2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/75b7dfce836541b1901f25dfe4043a15] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=99.3 K 2024-11-20T06:40:38,187 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:38,187 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/c926402bf75e4da8a04211015ce2d0de, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/45c4ca627c0b4cd8b0b3579eef684bb2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/75b7dfce836541b1901f25dfe4043a15] 2024-11-20T06:40:38,187 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting df565b904a62496d9c0895937a75d51f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732084832963 2024-11-20T06:40:38,189 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b22af5f28e144eefae793ff4236bf785, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732084834136 2024-11-20T06:40:38,190 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 111f36f47da847a3beb55e80f7da61b4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732084835301 2024-11-20T06:40:38,190 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting c926402bf75e4da8a04211015ce2d0de, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732084832963 2024-11-20T06:40:38,191 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c4ca627c0b4cd8b0b3579eef684bb2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732084834136 2024-11-20T06:40:38,191 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b7dfce836541b1901f25dfe4043a15, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732084835297 2024-11-20T06:40:38,200 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:38,202 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#B#compaction#145 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:38,203 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/6cf1dffc86764fc49cdf95448de29847 is 50, key is test_row_0/B:col10/1732084835301/Put/seqid=0 2024-11-20T06:40:38,204 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411204d5649530e8f46e988f1a49bd392ebb1_d735f3ed3d64a5ca97cafea9e7136d03 store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:38,207 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411204d5649530e8f46e988f1a49bd392ebb1_d735f3ed3d64a5ca97cafea9e7136d03, store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:38,207 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204d5649530e8f46e988f1a49bd392ebb1_d735f3ed3d64a5ca97cafea9e7136d03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:38,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741993_1169 (size=4469) 2024-11-20T06:40:38,231 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#A#compaction#144 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:38,232 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/f157dfeffbf9407e824fb41afd6b63c4 is 175, key is test_row_0/A:col10/1732084835301/Put/seqid=0 2024-11-20T06:40:38,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741992_1168 (size=12325) 2024-11-20T06:40:38,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741994_1170 (size=31279) 2024-11-20T06:40:38,242 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/6cf1dffc86764fc49cdf95448de29847 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/6cf1dffc86764fc49cdf95448de29847 2024-11-20T06:40:38,247 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/f157dfeffbf9407e824fb41afd6b63c4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/f157dfeffbf9407e824fb41afd6b63c4 2024-11-20T06:40:38,248 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/B of d735f3ed3d64a5ca97cafea9e7136d03 into 6cf1dffc86764fc49cdf95448de29847(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:38,248 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:38,248 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/B, priority=13, startTime=1732084838184; duration=0sec 2024-11-20T06:40:38,248 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:38,248 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:B 2024-11-20T06:40:38,248 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:38,252 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:38,252 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/C is initiating minor compaction (all files) 2024-11-20T06:40:38,252 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/C in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:38,253 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/074830ea7c474949acda70998efaaca4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/940d5aee2e124a10afe665308ef4322a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/d161b34b91fc47d2bfff1b517b977233] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=35.4 K 2024-11-20T06:40:38,254 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 074830ea7c474949acda70998efaaca4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732084832963 2024-11-20T06:40:38,255 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 940d5aee2e124a10afe665308ef4322a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732084834136 2024-11-20T06:40:38,255 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d161b34b91fc47d2bfff1b517b977233, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732084835301 2024-11-20T06:40:38,260 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/A of d735f3ed3d64a5ca97cafea9e7136d03 into f157dfeffbf9407e824fb41afd6b63c4(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:38,260 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:38,260 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/A, priority=13, startTime=1732084838184; duration=0sec 2024-11-20T06:40:38,260 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:38,260 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:A 2024-11-20T06:40:38,270 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:38,270 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/ff1dd14e31144eb3a0f69d5d77dd5e38 is 50, key is test_row_0/C:col10/1732084835301/Put/seqid=0 2024-11-20T06:40:38,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T06:40:38,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:38,280 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T06:40:38,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741995_1171 (size=12325) 2024-11-20T06:40:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201535636b822d4eb1a34ba1d0062e562b_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084836487/Put/seqid=0 2024-11-20T06:40:38,312 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/ff1dd14e31144eb3a0f69d5d77dd5e38 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ff1dd14e31144eb3a0f69d5d77dd5e38 2024-11-20T06:40:38,319 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/C of d735f3ed3d64a5ca97cafea9e7136d03 into ff1dd14e31144eb3a0f69d5d77dd5e38(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:38,319 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:38,319 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/C, priority=13, startTime=1732084838186; duration=0sec 2024-11-20T06:40:38,319 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:38,319 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:C 2024-11-20T06:40:38,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741996_1172 (size=12304) 2024-11-20T06:40:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:38,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:38,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084898661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084898661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084898664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084898665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084898665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:38,753 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201535636b822d4eb1a34ba1d0062e562b_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201535636b822d4eb1a34ba1d0062e562b_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:38,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/1140ea364d6242a5814a1199a2455320, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:38,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/1140ea364d6242a5814a1199a2455320 is 175, key is test_row_0/A:col10/1732084836487/Put/seqid=0 2024-11-20T06:40:38,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741997_1173 (size=31105) 2024-11-20T06:40:38,761 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/1140ea364d6242a5814a1199a2455320 2024-11-20T06:40:38,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/54b24fe7708d4c68a67cb3c49fc87c2c is 50, key is test_row_0/B:col10/1732084836487/Put/seqid=0 2024-11-20T06:40:38,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741998_1174 (size=12151) 2024-11-20T06:40:38,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084898774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084898774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084898775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084898775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084898775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084898984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084898984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084898984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084898984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084898984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T06:40:39,175 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/54b24fe7708d4c68a67cb3c49fc87c2c 2024-11-20T06:40:39,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/6d0b722d8c6d49eb84faf6ea05937964 is 50, key is test_row_0/C:col10/1732084836487/Put/seqid=0 2024-11-20T06:40:39,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741999_1175 (size=12151) 2024-11-20T06:40:39,201 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/6d0b722d8c6d49eb84faf6ea05937964 2024-11-20T06:40:39,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/1140ea364d6242a5814a1199a2455320 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1140ea364d6242a5814a1199a2455320 2024-11-20T06:40:39,216 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1140ea364d6242a5814a1199a2455320, entries=150, sequenceid=153, filesize=30.4 K 2024-11-20T06:40:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/54b24fe7708d4c68a67cb3c49fc87c2c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/54b24fe7708d4c68a67cb3c49fc87c2c 2024-11-20T06:40:39,221 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/54b24fe7708d4c68a67cb3c49fc87c2c, entries=150, sequenceid=153, filesize=11.9 K 2024-11-20T06:40:39,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/6d0b722d8c6d49eb84faf6ea05937964 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/6d0b722d8c6d49eb84faf6ea05937964 2024-11-20T06:40:39,228 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/6d0b722d8c6d49eb84faf6ea05937964, entries=150, sequenceid=153, filesize=11.9 K 2024-11-20T06:40:39,229 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for d735f3ed3d64a5ca97cafea9e7136d03 in 949ms, sequenceid=153, compaction requested=false 2024-11-20T06:40:39,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:39,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:39,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-20T06:40:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-20T06:40:39,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-20T06:40:39,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3320 sec 2024-11-20T06:40:39,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.3380 sec 2024-11-20T06:40:39,251 INFO [master/61c2066a8a0f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T06:40:39,252 INFO [master/61c2066a8a0f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T06:40:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:39,297 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:39,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T06:40:39,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cb0ae28328584b2bbf71d9023f85a4d8_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084839296/Put/seqid=0 2024-11-20T06:40:39,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742000_1176 (size=12304) 2024-11-20T06:40:39,316 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:39,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084899311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084899313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084899315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084899315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084899319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,322 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cb0ae28328584b2bbf71d9023f85a4d8_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cb0ae28328584b2bbf71d9023f85a4d8_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:39,324 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/ed2256d0945847a198cb1bc5b0283b4a, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:39,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/ed2256d0945847a198cb1bc5b0283b4a is 175, key is test_row_0/A:col10/1732084839296/Put/seqid=0 2024-11-20T06:40:39,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742001_1177 (size=31105) 2024-11-20T06:40:39,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084899424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084899424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084899424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084899424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084899424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084899634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084899634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084899634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084899634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084899634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,730 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/ed2256d0945847a198cb1bc5b0283b4a 2024-11-20T06:40:39,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/b80cdea77d884f198b8d29bc177cf631 is 50, key is test_row_0/B:col10/1732084839296/Put/seqid=0 2024-11-20T06:40:39,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742002_1178 (size=12151) 2024-11-20T06:40:39,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084899944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084899944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084899944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084899944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084899944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:39,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/b80cdea77d884f198b8d29bc177cf631 2024-11-20T06:40:40,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/4230eb8f129e48a2882dd451a71b9942 is 50, key is test_row_0/C:col10/1732084839296/Put/seqid=0 2024-11-20T06:40:40,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742003_1179 (size=12151) 2024-11-20T06:40:40,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/4230eb8f129e48a2882dd451a71b9942 2024-11-20T06:40:40,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/ed2256d0945847a198cb1bc5b0283b4a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ed2256d0945847a198cb1bc5b0283b4a 2024-11-20T06:40:40,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ed2256d0945847a198cb1bc5b0283b4a, entries=150, sequenceid=175, filesize=30.4 K 2024-11-20T06:40:40,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/b80cdea77d884f198b8d29bc177cf631 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b80cdea77d884f198b8d29bc177cf631 2024-11-20T06:40:40,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b80cdea77d884f198b8d29bc177cf631, entries=150, sequenceid=175, filesize=11.9 K 2024-11-20T06:40:40,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/4230eb8f129e48a2882dd451a71b9942 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/4230eb8f129e48a2882dd451a71b9942 2024-11-20T06:40:40,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/4230eb8f129e48a2882dd451a71b9942, entries=150, sequenceid=175, filesize=11.9 K 2024-11-20T06:40:40,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d735f3ed3d64a5ca97cafea9e7136d03 in 901ms, sequenceid=175, compaction requested=true 2024-11-20T06:40:40,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:40,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:40,199 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:40,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:40,199 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:40,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:40,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:40,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:40,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:40,200 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:40,200 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93489 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:40,200 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/B is initiating minor compaction (all files) 2024-11-20T06:40:40,200 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/A is initiating minor compaction (all files) 2024-11-20T06:40:40,200 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/A in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:40,200 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/f157dfeffbf9407e824fb41afd6b63c4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1140ea364d6242a5814a1199a2455320, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ed2256d0945847a198cb1bc5b0283b4a] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=91.3 K 2024-11-20T06:40:40,200 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:40,200 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/f157dfeffbf9407e824fb41afd6b63c4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1140ea364d6242a5814a1199a2455320, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ed2256d0945847a198cb1bc5b0283b4a] 2024-11-20T06:40:40,200 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/B in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:40,200 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/6cf1dffc86764fc49cdf95448de29847, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/54b24fe7708d4c68a67cb3c49fc87c2c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b80cdea77d884f198b8d29bc177cf631] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=35.8 K 2024-11-20T06:40:40,201 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cf1dffc86764fc49cdf95448de29847, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732084835301 2024-11-20T06:40:40,201 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting f157dfeffbf9407e824fb41afd6b63c4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732084835301 2024-11-20T06:40:40,201 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 54b24fe7708d4c68a67cb3c49fc87c2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732084836487 2024-11-20T06:40:40,201 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1140ea364d6242a5814a1199a2455320, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732084836487 2024-11-20T06:40:40,201 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b80cdea77d884f198b8d29bc177cf631, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732084839296 2024-11-20T06:40:40,202 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed2256d0945847a198cb1bc5b0283b4a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732084839296 2024-11-20T06:40:40,208 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:40,209 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:40,209 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/a14dc13de642443babb894a97a375e51 is 50, key is test_row_0/B:col10/1732084839296/Put/seqid=0 2024-11-20T06:40:40,215 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120155ba3814ad1470dba11f483c5a6bb37_d735f3ed3d64a5ca97cafea9e7136d03 store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:40,218 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120155ba3814ad1470dba11f483c5a6bb37_d735f3ed3d64a5ca97cafea9e7136d03, store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:40,218 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120155ba3814ad1470dba11f483c5a6bb37_d735f3ed3d64a5ca97cafea9e7136d03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:40,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742004_1180 (size=12527) 2024-11-20T06:40:40,229 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/a14dc13de642443babb894a97a375e51 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a14dc13de642443babb894a97a375e51 2024-11-20T06:40:40,240 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/B of d735f3ed3d64a5ca97cafea9e7136d03 into a14dc13de642443babb894a97a375e51(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:40,240 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:40,240 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/B, priority=13, startTime=1732084840199; duration=0sec 2024-11-20T06:40:40,240 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:40,240 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:B 2024-11-20T06:40:40,240 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:40:40,243 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:40:40,243 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/C is initiating minor compaction (all files) 2024-11-20T06:40:40,243 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/C in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:40,243 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ff1dd14e31144eb3a0f69d5d77dd5e38, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/6d0b722d8c6d49eb84faf6ea05937964, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/4230eb8f129e48a2882dd451a71b9942] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=35.8 K 2024-11-20T06:40:40,243 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1dd14e31144eb3a0f69d5d77dd5e38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732084835301 2024-11-20T06:40:40,244 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0b722d8c6d49eb84faf6ea05937964, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732084836487 2024-11-20T06:40:40,244 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 4230eb8f129e48a2882dd451a71b9942, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732084839296 2024-11-20T06:40:40,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742005_1181 (size=4469) 2024-11-20T06:40:40,249 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#A#compaction#154 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:40,250 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/6e70ffecebf34328a0547bba9e44e29d is 175, key is test_row_0/A:col10/1732084839296/Put/seqid=0 2024-11-20T06:40:40,254 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:40,255 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/397cd89fb0424623b3eff8ac2bd728ae is 50, key is test_row_0/C:col10/1732084839296/Put/seqid=0 2024-11-20T06:40:40,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742006_1182 (size=31481) 2024-11-20T06:40:40,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742007_1183 (size=12527) 2024-11-20T06:40:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:40,456 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:40,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T06:40:40,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120baa6f621c0954abe8f27af331b6a744d_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084840455/Put/seqid=0 2024-11-20T06:40:40,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742008_1184 (size=17284) 2024-11-20T06:40:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084900472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084900473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084900474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084900475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084900476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084900584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084900584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084900584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084900584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084900584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,670 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/6e70ffecebf34328a0547bba9e44e29d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6e70ffecebf34328a0547bba9e44e29d 2024-11-20T06:40:40,670 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/397cd89fb0424623b3eff8ac2bd728ae as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/397cd89fb0424623b3eff8ac2bd728ae 2024-11-20T06:40:40,676 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/A of d735f3ed3d64a5ca97cafea9e7136d03 into 6e70ffecebf34328a0547bba9e44e29d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:40,676 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:40,677 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/A, priority=13, startTime=1732084840198; duration=0sec 2024-11-20T06:40:40,677 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:40,677 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:A 2024-11-20T06:40:40,678 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/C of d735f3ed3d64a5ca97cafea9e7136d03 into 397cd89fb0424623b3eff8ac2bd728ae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:40,678 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:40,678 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/C, priority=13, startTime=1732084840199; duration=0sec 2024-11-20T06:40:40,678 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:40,678 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:C 2024-11-20T06:40:40,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084900794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084900794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084900794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084900794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084900794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:40,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:40,873 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120baa6f621c0954abe8f27af331b6a744d_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120baa6f621c0954abe8f27af331b6a744d_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:40,874 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/8791820e018c41989e7182515c42973b, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:40,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/8791820e018c41989e7182515c42973b is 175, key is test_row_0/A:col10/1732084840455/Put/seqid=0 2024-11-20T06:40:40,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742009_1185 (size=48389) 2024-11-20T06:40:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T06:40:41,084 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:41,084 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-20T06:40:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T06:40:41,085 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:41,086 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:41,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:41,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:41,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084901104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:41,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084901104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084901104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:41,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:41,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084901104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:41,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:41,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:41,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084901105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:41,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T06:40:41,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T06:40:41,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:41,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:41,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:41,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,280 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/8791820e018c41989e7182515c42973b 2024-11-20T06:40:41,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/b35bc11ad7004a14a658f6d420565540 is 50, key is test_row_0/B:col10/1732084840455/Put/seqid=0 2024-11-20T06:40:41,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742010_1186 (size=12151) 2024-11-20T06:40:41,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T06:40:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:41,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T06:40:41,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T06:40:41,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:41,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:41,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:41,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:41,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084901614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084901614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:41,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:41,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:41,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084901614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084901614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:41,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084901615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:41,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/b35bc11ad7004a14a658f6d420565540 2024-11-20T06:40:41,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T06:40:41,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:41,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:41,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:41,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:41,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/c66426fcb8104c9bb28da55f930e98c6 is 50, key is test_row_0/C:col10/1732084840455/Put/seqid=0 2024-11-20T06:40:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T06:40:41,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742011_1187 (size=12151) 2024-11-20T06:40:41,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/c66426fcb8104c9bb28da55f930e98c6 2024-11-20T06:40:41,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/8791820e018c41989e7182515c42973b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/8791820e018c41989e7182515c42973b 2024-11-20T06:40:41,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/8791820e018c41989e7182515c42973b, entries=250, sequenceid=193, filesize=47.3 K 2024-11-20T06:40:41,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/b35bc11ad7004a14a658f6d420565540 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b35bc11ad7004a14a658f6d420565540 2024-11-20T06:40:41,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b35bc11ad7004a14a658f6d420565540, entries=150, sequenceid=193, filesize=11.9 K 2024-11-20T06:40:41,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/c66426fcb8104c9bb28da55f930e98c6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/c66426fcb8104c9bb28da55f930e98c6 2024-11-20T06:40:41,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/c66426fcb8104c9bb28da55f930e98c6, entries=150, sequenceid=193, filesize=11.9 K 2024-11-20T06:40:41,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for d735f3ed3d64a5ca97cafea9e7136d03 in 1315ms, sequenceid=193, compaction requested=false 2024-11-20T06:40:41,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:41,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T06:40:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:41,859 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T06:40:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202621ea51342e4ee287aa095977f371bd_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084840474/Put/seqid=0 2024-11-20T06:40:41,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742012_1188 (size=12304) 2024-11-20T06:40:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T06:40:42,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:42,285 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202621ea51342e4ee287aa095977f371bd_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202621ea51342e4ee287aa095977f371bd_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/9e167cf0ba2f49cfa0f1ce9a902225ea, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/9e167cf0ba2f49cfa0f1ce9a902225ea is 175, key is test_row_0/A:col10/1732084840474/Put/seqid=0 2024-11-20T06:40:42,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742013_1189 (size=31105) 2024-11-20T06:40:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:42,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:42,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084902653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:42,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084902655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:42,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084902655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084902656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:42,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084902656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:42,697 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/9e167cf0ba2f49cfa0f1ce9a902225ea 2024-11-20T06:40:42,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/704e61f46ac6454ca8f74cd7a60a8720 is 50, key is test_row_0/B:col10/1732084840474/Put/seqid=0 2024-11-20T06:40:42,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742014_1190 (size=12151) 2024-11-20T06:40:42,737 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/704e61f46ac6454ca8f74cd7a60a8720 2024-11-20T06:40:42,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/568c20e5e8dd464798601fba7f45fdfe is 50, key is test_row_0/C:col10/1732084840474/Put/seqid=0 2024-11-20T06:40:42,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084902764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:42,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084902764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084902764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084902764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084902764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:42,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742015_1191 (size=12151) 2024-11-20T06:40:42,786 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/568c20e5e8dd464798601fba7f45fdfe 2024-11-20T06:40:42,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/9e167cf0ba2f49cfa0f1ce9a902225ea as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/9e167cf0ba2f49cfa0f1ce9a902225ea 2024-11-20T06:40:42,800 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/9e167cf0ba2f49cfa0f1ce9a902225ea, entries=150, sequenceid=214, filesize=30.4 K 2024-11-20T06:40:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/704e61f46ac6454ca8f74cd7a60a8720 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/704e61f46ac6454ca8f74cd7a60a8720 2024-11-20T06:40:42,814 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/704e61f46ac6454ca8f74cd7a60a8720, entries=150, sequenceid=214, filesize=11.9 K 2024-11-20T06:40:42,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/568c20e5e8dd464798601fba7f45fdfe as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/568c20e5e8dd464798601fba7f45fdfe 2024-11-20T06:40:42,825 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/568c20e5e8dd464798601fba7f45fdfe, entries=150, sequenceid=214, filesize=11.9 K 2024-11-20T06:40:42,826 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for d735f3ed3d64a5ca97cafea9e7136d03 in 967ms, sequenceid=214, compaction requested=true 2024-11-20T06:40:42,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:42,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:42,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-20T06:40:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-20T06:40:42,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-20T06:40:42,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7420 sec 2024-11-20T06:40:42,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.7470 sec 2024-11-20T06:40:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:42,987 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:42,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T06:40:43,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120be23ebd62eaa4387b8ec29a8004a65cf_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084842653/Put/seqid=0 2024-11-20T06:40:43,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742016_1192 (size=14794) 2024-11-20T06:40:43,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084903011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,018 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,023 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120be23ebd62eaa4387b8ec29a8004a65cf_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be23ebd62eaa4387b8ec29a8004a65cf_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:43,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084903017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,024 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/6b9ef52dbfb842d98191033a5f5c52a3, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:43,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084903017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/6b9ef52dbfb842d98191033a5f5c52a3 is 175, key is test_row_0/A:col10/1732084842653/Put/seqid=0 2024-11-20T06:40:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084903018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084903019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742017_1193 (size=39749) 2024-11-20T06:40:43,054 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/6b9ef52dbfb842d98191033a5f5c52a3 2024-11-20T06:40:43,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/db89f050759e4256b5bf2ffdd8ab0e85 is 50, key is test_row_0/B:col10/1732084842653/Put/seqid=0 2024-11-20T06:40:43,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742018_1194 (size=12151) 2024-11-20T06:40:43,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/db89f050759e4256b5bf2ffdd8ab0e85 2024-11-20T06:40:43,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/48b87c4548764ac4b565285599f8417d is 50, key is test_row_0/C:col10/1732084842653/Put/seqid=0 2024-11-20T06:40:43,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742019_1195 (size=12151) 2024-11-20T06:40:43,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/48b87c4548764ac4b565285599f8417d 2024-11-20T06:40:43,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/6b9ef52dbfb842d98191033a5f5c52a3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6b9ef52dbfb842d98191033a5f5c52a3 2024-11-20T06:40:43,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6b9ef52dbfb842d98191033a5f5c52a3, entries=200, sequenceid=233, filesize=38.8 K 2024-11-20T06:40:43,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/db89f050759e4256b5bf2ffdd8ab0e85 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/db89f050759e4256b5bf2ffdd8ab0e85 2024-11-20T06:40:43,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/db89f050759e4256b5bf2ffdd8ab0e85, entries=150, sequenceid=233, filesize=11.9 K 2024-11-20T06:40:43,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/48b87c4548764ac4b565285599f8417d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/48b87c4548764ac4b565285599f8417d 2024-11-20T06:40:43,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084903124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/48b87c4548764ac4b565285599f8417d, entries=150, sequenceid=233, filesize=11.9 K 2024-11-20T06:40:43,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for d735f3ed3d64a5ca97cafea9e7136d03 in 152ms, sequenceid=233, compaction requested=true 2024-11-20T06:40:43,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:43,139 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:43,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:43,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:43,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:43,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:43,140 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:43,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:43,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:43,141 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150724 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:43,141 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/A is initiating minor compaction (all files) 2024-11-20T06:40:43,141 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/A in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,142 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6e70ffecebf34328a0547bba9e44e29d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/8791820e018c41989e7182515c42973b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/9e167cf0ba2f49cfa0f1ce9a902225ea, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6b9ef52dbfb842d98191033a5f5c52a3] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=147.2 K 2024-11-20T06:40:43,142 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:43,142 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6e70ffecebf34328a0547bba9e44e29d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/8791820e018c41989e7182515c42973b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/9e167cf0ba2f49cfa0f1ce9a902225ea, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6b9ef52dbfb842d98191033a5f5c52a3] 2024-11-20T06:40:43,142 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:43,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:40:43,143 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:43,143 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/B is initiating minor compaction (all files) 2024-11-20T06:40:43,144 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/B in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,144 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a14dc13de642443babb894a97a375e51, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b35bc11ad7004a14a658f6d420565540, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/704e61f46ac6454ca8f74cd7a60a8720, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/db89f050759e4256b5bf2ffdd8ab0e85] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=47.8 K 2024-11-20T06:40:43,144 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e70ffecebf34328a0547bba9e44e29d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732084839296 2024-11-20T06:40:43,145 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a14dc13de642443babb894a97a375e51, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732084839296 2024-11-20T06:40:43,145 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8791820e018c41989e7182515c42973b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732084839305 2024-11-20T06:40:43,145 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b35bc11ad7004a14a658f6d420565540, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732084839311 2024-11-20T06:40:43,145 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e167cf0ba2f49cfa0f1ce9a902225ea, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732084840471 2024-11-20T06:40:43,146 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 704e61f46ac6454ca8f74cd7a60a8720, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732084840471 2024-11-20T06:40:43,146 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b9ef52dbfb842d98191033a5f5c52a3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732084842653 2024-11-20T06:40:43,146 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting db89f050759e4256b5bf2ffdd8ab0e85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732084842653 2024-11-20T06:40:43,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084903161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084903167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084903169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084903172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,191 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:43,192 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/249c01d44b5a4f43b652288e0ad2115e is 50, key is test_row_0/B:col10/1732084842653/Put/seqid=0 2024-11-20T06:40:43,203 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:43,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120de655e0e92f24e0cb47e984aabe71d42_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084843140/Put/seqid=0 2024-11-20T06:40:43,231 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112064178c5ef9804250b607ac57dd5c3f3a_d735f3ed3d64a5ca97cafea9e7136d03 store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:43,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T06:40:43,234 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112064178c5ef9804250b607ac57dd5c3f3a_d735f3ed3d64a5ca97cafea9e7136d03, store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:43,235 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:43,235 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-20T06:40:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T06:40:43,237 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:43,238 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112064178c5ef9804250b607ac57dd5c3f3a_d735f3ed3d64a5ca97cafea9e7136d03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:43,240 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:43,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:43,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742020_1196 (size=12663) 2024-11-20T06:40:43,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742022_1198 (size=4469) 2024-11-20T06:40:43,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742021_1197 (size=17284) 2024-11-20T06:40:43,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084903273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084903284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084903284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084903289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T06:40:43,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084903344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084903483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084903494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084903494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084903494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:43,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:43,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T06:40:43,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084903654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,668 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#A#compaction#166 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:43,669 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/e5cd654adf2f4fc7884156996431b197 is 175, key is test_row_0/A:col10/1732084842653/Put/seqid=0 2024-11-20T06:40:43,669 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/249c01d44b5a4f43b652288e0ad2115e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/249c01d44b5a4f43b652288e0ad2115e 2024-11-20T06:40:43,671 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:43,675 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120de655e0e92f24e0cb47e984aabe71d42_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de655e0e92f24e0cb47e984aabe71d42_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:43,676 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/B of d735f3ed3d64a5ca97cafea9e7136d03 into 249c01d44b5a4f43b652288e0ad2115e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:43,676 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:43,676 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/B, priority=12, startTime=1732084843139; duration=0sec 2024-11-20T06:40:43,676 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:43,676 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:B 2024-11-20T06:40:43,676 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:43,676 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/64cc5fd704fb4f0288848198881f90dd, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:43,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/64cc5fd704fb4f0288848198881f90dd is 175, key is test_row_0/A:col10/1732084843140/Put/seqid=0 2024-11-20T06:40:43,678 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:43,678 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/C is initiating minor compaction (all files) 2024-11-20T06:40:43,678 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/C in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,678 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/397cd89fb0424623b3eff8ac2bd728ae, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/c66426fcb8104c9bb28da55f930e98c6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/568c20e5e8dd464798601fba7f45fdfe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/48b87c4548764ac4b565285599f8417d] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=47.8 K 2024-11-20T06:40:43,681 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 397cd89fb0424623b3eff8ac2bd728ae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732084839296 2024-11-20T06:40:43,688 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c66426fcb8104c9bb28da55f930e98c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732084839311 2024-11-20T06:40:43,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742023_1199 (size=31617) 2024-11-20T06:40:43,690 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 568c20e5e8dd464798601fba7f45fdfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732084840471 2024-11-20T06:40:43,693 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 48b87c4548764ac4b565285599f8417d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732084842653 2024-11-20T06:40:43,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742024_1200 (size=48389) 2024-11-20T06:40:43,697 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/e5cd654adf2f4fc7884156996431b197 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/e5cd654adf2f4fc7884156996431b197 2024-11-20T06:40:43,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:43,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:43,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,710 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/A of d735f3ed3d64a5ca97cafea9e7136d03 into e5cd654adf2f4fc7884156996431b197(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:43,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:43,711 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/A, priority=12, startTime=1732084843139; duration=0sec 2024-11-20T06:40:43,712 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:43,712 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:A 2024-11-20T06:40:43,732 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:43,732 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/b900f4c4d16b4127a5e904c03068cea5 is 50, key is test_row_0/C:col10/1732084842653/Put/seqid=0 2024-11-20T06:40:43,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742025_1201 (size=12663) 2024-11-20T06:40:43,795 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/b900f4c4d16b4127a5e904c03068cea5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/b900f4c4d16b4127a5e904c03068cea5 2024-11-20T06:40:43,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084903794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,803 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/C of d735f3ed3d64a5ca97cafea9e7136d03 into b900f4c4d16b4127a5e904c03068cea5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:43,803 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:43,803 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/C, priority=12, startTime=1732084843140; duration=0sec 2024-11-20T06:40:43,803 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:43,803 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:C 2024-11-20T06:40:43,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084903804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084903804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084903805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:43,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:43,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:43,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:43,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T06:40:44,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:44,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:44,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/64cc5fd704fb4f0288848198881f90dd 2024-11-20T06:40:44,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/d8f1ef522062437fbef4b5dd08e4ba55 is 50, key is test_row_0/B:col10/1732084843140/Put/seqid=0 2024-11-20T06:40:44,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742026_1202 (size=12151) 2024-11-20T06:40:44,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:44,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:44,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084904175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:44,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084904304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:44,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084904314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:44,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084904314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:44,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084904324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:44,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:44,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:44,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T06:40:44,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:44,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/d8f1ef522062437fbef4b5dd08e4ba55 2024-11-20T06:40:44,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/72ba3d8755404019a476f5feea363c4e is 50, key is test_row_0/C:col10/1732084843140/Put/seqid=0 2024-11-20T06:40:44,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742027_1203 (size=12151) 2024-11-20T06:40:44,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:44,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:44,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:44,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:44,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:44,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:44,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:44,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:44,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/72ba3d8755404019a476f5feea363c4e 2024-11-20T06:40:44,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/64cc5fd704fb4f0288848198881f90dd as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/64cc5fd704fb4f0288848198881f90dd 2024-11-20T06:40:44,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/64cc5fd704fb4f0288848198881f90dd, entries=250, sequenceid=255, filesize=47.3 K 2024-11-20T06:40:44,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/d8f1ef522062437fbef4b5dd08e4ba55 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/d8f1ef522062437fbef4b5dd08e4ba55 2024-11-20T06:40:44,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/d8f1ef522062437fbef4b5dd08e4ba55, entries=150, sequenceid=255, filesize=11.9 K 2024-11-20T06:40:44,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/72ba3d8755404019a476f5feea363c4e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/72ba3d8755404019a476f5feea363c4e 2024-11-20T06:40:45,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/72ba3d8755404019a476f5feea363c4e, entries=150, sequenceid=255, filesize=11.9 K 2024-11-20T06:40:45,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d735f3ed3d64a5ca97cafea9e7136d03 in 1863ms, sequenceid=255, compaction requested=false 2024-11-20T06:40:45,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:45,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T06:40:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:45,093 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T06:40:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203b0c4ffb76a241b08b689d9be973ea6f_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084843166/Put/seqid=0 2024-11-20T06:40:45,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742028_1204 (size=12454) 2024-11-20T06:40:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:45,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:45,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:45,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084905297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:45,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084905313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:45,318 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in d735f3ed3d64a5ca97cafea9e7136d03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:45,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084905334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:45,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084905334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:45,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084905334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:45,339 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in d735f3ed3d64a5ca97cafea9e7136d03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:45,339 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in d735f3ed3d64a5ca97cafea9e7136d03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:45,340 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in d735f3ed3d64a5ca97cafea9e7136d03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T06:40:45,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084905404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:45,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:45,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:45,530 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203b0c4ffb76a241b08b689d9be973ea6f_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203b0c4ffb76a241b08b689d9be973ea6f_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:45,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/b27d9849bb404a2bba571f8e0af3f7ea, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/b27d9849bb404a2bba571f8e0af3f7ea is 175, key is test_row_0/A:col10/1732084843166/Put/seqid=0 2024-11-20T06:40:45,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742029_1205 (size=31255) 2024-11-20T06:40:45,549 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/b27d9849bb404a2bba571f8e0af3f7ea 2024-11-20T06:40:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/8783209583ac4a9f9d004312b6037a21 is 50, key is test_row_0/B:col10/1732084843166/Put/seqid=0 2024-11-20T06:40:45,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742030_1206 (size=12301) 2024-11-20T06:40:45,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084905614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:45,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084905924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:46,007 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/8783209583ac4a9f9d004312b6037a21 2024-11-20T06:40:46,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/5ca0c8ba49314373ab8b2619cd332c98 is 50, key is test_row_0/C:col10/1732084843166/Put/seqid=0 2024-11-20T06:40:46,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742031_1207 (size=12301) 2024-11-20T06:40:46,432 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/5ca0c8ba49314373ab8b2619cd332c98 2024-11-20T06:40:46,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084906433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:46,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:46,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/b27d9849bb404a2bba571f8e0af3f7ea as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/b27d9849bb404a2bba571f8e0af3f7ea 2024-11-20T06:40:46,441 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/b27d9849bb404a2bba571f8e0af3f7ea, entries=150, sequenceid=272, filesize=30.5 K 2024-11-20T06:40:46,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/8783209583ac4a9f9d004312b6037a21 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/8783209583ac4a9f9d004312b6037a21 2024-11-20T06:40:46,446 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/8783209583ac4a9f9d004312b6037a21, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T06:40:46,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/5ca0c8ba49314373ab8b2619cd332c98 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/5ca0c8ba49314373ab8b2619cd332c98 2024-11-20T06:40:46,451 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/5ca0c8ba49314373ab8b2619cd332c98, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T06:40:46,451 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d735f3ed3d64a5ca97cafea9e7136d03 in 1358ms, sequenceid=272, compaction requested=true 2024-11-20T06:40:46,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:46,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:46,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-20T06:40:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-20T06:40:46,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-20T06:40:46,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2130 sec 2024-11-20T06:40:46,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 3.2190 sec 2024-11-20T06:40:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:47,325 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:47,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T06:40:47,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cee06e7a8de840d9b1a07de88a5599aa_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084845280/Put/seqid=0 2024-11-20T06:40:47,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742032_1208 (size=14994) 2024-11-20T06:40:47,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084907358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:47,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:47,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42966 deadline: 1732084907363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43020 deadline: 1732084907363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:47,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43008 deadline: 1732084907363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:47,366 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in d735f3ed3d64a5ca97cafea9e7136d03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:47,366 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in d735f3ed3d64a5ca97cafea9e7136d03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:47,367 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in d735f3ed3d64a5ca97cafea9e7136d03 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:47,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T06:40:47,405 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:47,405 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-20T06:40:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T06:40:47,407 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:47,408 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:47,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:47,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42996 deadline: 1732084907444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:47,448 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in d735f3ed3d64a5ca97cafea9e7136d03 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:40:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:47,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084907474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T06:40:47,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:47,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:47,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084907684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:47,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:47,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:47,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:47,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:47,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:47,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T06:40:47,738 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:47,742 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cee06e7a8de840d9b1a07de88a5599aa_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cee06e7a8de840d9b1a07de88a5599aa_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:47,743 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/7e2e193537274a60817fb0ebc91b7427, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:47,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/7e2e193537274a60817fb0ebc91b7427 is 175, key is test_row_0/A:col10/1732084845280/Put/seqid=0 2024-11-20T06:40:47,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742033_1209 (size=39949) 2024-11-20T06:40:47,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:47,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:47,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:47,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:47,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:47,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:47,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084907993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:47,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:48,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:48,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:48,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T06:40:48,150 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/7e2e193537274a60817fb0ebc91b7427 2024-11-20T06:40:48,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/a0da46f1169f4724bc53991e0a649200 is 50, key is test_row_0/B:col10/1732084845280/Put/seqid=0 2024-11-20T06:40:48,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742034_1210 (size=12301) 2024-11-20T06:40:48,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:48,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:48,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:48,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:48,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42962 deadline: 1732084908504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:48,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:48,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:48,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:40:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T06:40:48,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/a0da46f1169f4724bc53991e0a649200 2024-11-20T06:40:48,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/51eb61f989a64ce0b77402abcae4b09a is 50, key is test_row_0/C:col10/1732084845280/Put/seqid=0 2024-11-20T06:40:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742035_1211 (size=12301) 2024-11-20T06:40:48,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:48,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:48,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:48,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:48,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:48,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:48,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/51eb61f989a64ce0b77402abcae4b09a 2024-11-20T06:40:48,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/7e2e193537274a60817fb0ebc91b7427 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/7e2e193537274a60817fb0ebc91b7427 2024-11-20T06:40:48,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/7e2e193537274a60817fb0ebc91b7427, entries=200, sequenceid=295, filesize=39.0 K 2024-11-20T06:40:48,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/a0da46f1169f4724bc53991e0a649200 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a0da46f1169f4724bc53991e0a649200 2024-11-20T06:40:48,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a0da46f1169f4724bc53991e0a649200, entries=150, sequenceid=295, filesize=12.0 K 2024-11-20T06:40:48,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/51eb61f989a64ce0b77402abcae4b09a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/51eb61f989a64ce0b77402abcae4b09a 2024-11-20T06:40:48,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/51eb61f989a64ce0b77402abcae4b09a, entries=150, sequenceid=295, filesize=12.0 K 2024-11-20T06:40:49,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d735f3ed3d64a5ca97cafea9e7136d03 in 1697ms, sequenceid=295, compaction requested=true 2024-11-20T06:40:49,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:49,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:40:49,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:49,022 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:49,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:40:49,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:49,022 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:49,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d735f3ed3d64a5ca97cafea9e7136d03:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:40:49,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:49,025 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:49,025 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151210 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:49,026 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/A is initiating minor compaction (all files) 2024-11-20T06:40:49,026 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/B is initiating minor compaction (all files) 2024-11-20T06:40:49,026 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/A in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:49,026 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/B in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:49,026 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/249c01d44b5a4f43b652288e0ad2115e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/d8f1ef522062437fbef4b5dd08e4ba55, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/8783209583ac4a9f9d004312b6037a21, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a0da46f1169f4724bc53991e0a649200] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=48.3 K 2024-11-20T06:40:49,026 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/e5cd654adf2f4fc7884156996431b197, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/64cc5fd704fb4f0288848198881f90dd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/b27d9849bb404a2bba571f8e0af3f7ea, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/7e2e193537274a60817fb0ebc91b7427] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=147.7 K 2024-11-20T06:40:49,026 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:49,026 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/e5cd654adf2f4fc7884156996431b197, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/64cc5fd704fb4f0288848198881f90dd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/b27d9849bb404a2bba571f8e0af3f7ea, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/7e2e193537274a60817fb0ebc91b7427] 2024-11-20T06:40:49,026 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 249c01d44b5a4f43b652288e0ad2115e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732084842653 2024-11-20T06:40:49,026 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5cd654adf2f4fc7884156996431b197, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732084842653 2024-11-20T06:40:49,026 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d8f1ef522062437fbef4b5dd08e4ba55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732084843016 2024-11-20T06:40:49,027 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64cc5fd704fb4f0288848198881f90dd, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732084843009 2024-11-20T06:40:49,027 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8783209583ac4a9f9d004312b6037a21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732084843159 2024-11-20T06:40:49,027 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27d9849bb404a2bba571f8e0af3f7ea, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732084843159 2024-11-20T06:40:49,027 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a0da46f1169f4724bc53991e0a649200, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732084845280 2024-11-20T06:40:49,027 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e2e193537274a60817fb0ebc91b7427, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732084845280 2024-11-20T06:40:49,033 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:49,035 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:49,036 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/a09317b2b76d4f9ea8bd1d1136cabf62 is 50, key is test_row_0/B:col10/1732084845280/Put/seqid=0 2024-11-20T06:40:49,047 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411204eebb6e5d32d493083850bb19b20bba2_d735f3ed3d64a5ca97cafea9e7136d03 store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:49,050 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411204eebb6e5d32d493083850bb19b20bba2_d735f3ed3d64a5ca97cafea9e7136d03, store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:49,050 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204eebb6e5d32d493083850bb19b20bba2_d735f3ed3d64a5ca97cafea9e7136d03 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:49,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742036_1212 (size=12949) 2024-11-20T06:40:49,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742037_1213 (size=4469) 2024-11-20T06:40:49,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-20T06:40:49,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:49,090 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T06:40:49,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120210716ad9ac24f75b5e552a6a6581b3e_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084847357/Put/seqid=0 2024-11-20T06:40:49,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742038_1214 (size=12454) 2024-11-20T06:40:49,327 INFO [Thread-683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-683. 2024-11-20T06:40:49,327 DEBUG [Thread-683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:49,327 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,327 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:49,329 INFO [Thread-685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-685. 2024-11-20T06:40:49,329 DEBUG [Thread-685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:49,329 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,329 INFO [Thread-681 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-681. 2024-11-20T06:40:49,329 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,329 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:49,329 DEBUG [Thread-681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:49,329 DEBUG [Thread-681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,330 DEBUG [Thread-681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:49,332 INFO [Thread-687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-687. 2024-11-20T06:40:49,332 DEBUG [Thread-687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:49,332 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,332 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:49,466 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/a09317b2b76d4f9ea8bd1d1136cabf62 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a09317b2b76d4f9ea8bd1d1136cabf62 2024-11-20T06:40:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:49,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. as already flushing 2024-11-20T06:40:49,467 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-20T06:40:49,467 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:49,467 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,467 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:49,471 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/B of d735f3ed3d64a5ca97cafea9e7136d03 into a09317b2b76d4f9ea8bd1d1136cabf62(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:49,471 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:49,472 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/B, priority=12, startTime=1732084849022; duration=0sec 2024-11-20T06:40:49,472 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:40:49,472 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:B 2024-11-20T06:40:49,472 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:40:49,473 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:40:49,473 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): d735f3ed3d64a5ca97cafea9e7136d03/C is initiating minor compaction (all files) 2024-11-20T06:40:49,473 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d735f3ed3d64a5ca97cafea9e7136d03/C in TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:49,473 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/b900f4c4d16b4127a5e904c03068cea5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/72ba3d8755404019a476f5feea363c4e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/5ca0c8ba49314373ab8b2619cd332c98, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/51eb61f989a64ce0b77402abcae4b09a] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp, totalSize=48.3 K 2024-11-20T06:40:49,474 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b900f4c4d16b4127a5e904c03068cea5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732084842653 2024-11-20T06:40:49,474 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ba3d8755404019a476f5feea363c4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732084843016 2024-11-20T06:40:49,474 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca0c8ba49314373ab8b2619cd332c98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732084843159 2024-11-20T06:40:49,474 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 51eb61f989a64ce0b77402abcae4b09a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732084845280 2024-11-20T06:40:49,476 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:49,477 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/aa7f56991b1a4f4a9a8071667f8ddb3e is 175, key is test_row_0/A:col10/1732084845280/Put/seqid=0 2024-11-20T06:40:49,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742039_1215 (size=31903) 2024-11-20T06:40:49,483 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d735f3ed3d64a5ca97cafea9e7136d03#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:40:49,483 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/de7f8fd21c2d46b1a45017af13f91005 is 50, key is test_row_0/C:col10/1732084845280/Put/seqid=0 2024-11-20T06:40:49,485 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/aa7f56991b1a4f4a9a8071667f8ddb3e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/aa7f56991b1a4f4a9a8071667f8ddb3e 2024-11-20T06:40:49,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742040_1216 (size=12949) 2024-11-20T06:40:49,490 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/A of d735f3ed3d64a5ca97cafea9e7136d03 into aa7f56991b1a4f4a9a8071667f8ddb3e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:49,490 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:49,490 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/A, priority=12, startTime=1732084849022; duration=0sec 2024-11-20T06:40:49,490 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:49,490 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:A 2024-11-20T06:40:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:49,515 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120210716ad9ac24f75b5e552a6a6581b3e_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120210716ad9ac24f75b5e552a6a6581b3e_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/1110e4877b13434191dfd4f14ee4c8e0, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:49,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/1110e4877b13434191dfd4f14ee4c8e0 is 175, key is test_row_0/A:col10/1732084847357/Put/seqid=0 2024-11-20T06:40:49,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742041_1217 (size=31255) 2024-11-20T06:40:49,526 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-11-20T06:40:49,526 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:49,526 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,526 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:49,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T06:40:49,895 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/de7f8fd21c2d46b1a45017af13f91005 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/de7f8fd21c2d46b1a45017af13f91005 2024-11-20T06:40:49,902 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d735f3ed3d64a5ca97cafea9e7136d03/C of d735f3ed3d64a5ca97cafea9e7136d03 into de7f8fd21c2d46b1a45017af13f91005(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:40:49,902 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:49,902 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., storeName=d735f3ed3d64a5ca97cafea9e7136d03/C, priority=12, startTime=1732084849022; duration=0sec 2024-11-20T06:40:49,903 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:40:49,903 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d735f3ed3d64a5ca97cafea9e7136d03:C 2024-11-20T06:40:49,923 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/1110e4877b13434191dfd4f14ee4c8e0 2024-11-20T06:40:49,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/8519425fce7b494c99d331d8cf0a859e is 50, key is test_row_0/B:col10/1732084847357/Put/seqid=0 2024-11-20T06:40:49,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742042_1218 (size=12301) 2024-11-20T06:40:50,341 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/8519425fce7b494c99d331d8cf0a859e 2024-11-20T06:40:50,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/a5d35443253f4138932ba1d7115028f9 is 50, key is test_row_0/C:col10/1732084847357/Put/seqid=0 2024-11-20T06:40:50,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742043_1219 (size=12301) 2024-11-20T06:40:50,759 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/a5d35443253f4138932ba1d7115028f9 2024-11-20T06:40:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/1110e4877b13434191dfd4f14ee4c8e0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1110e4877b13434191dfd4f14ee4c8e0 2024-11-20T06:40:50,773 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1110e4877b13434191dfd4f14ee4c8e0, entries=150, sequenceid=308, filesize=30.5 K 2024-11-20T06:40:50,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/8519425fce7b494c99d331d8cf0a859e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/8519425fce7b494c99d331d8cf0a859e 2024-11-20T06:40:50,778 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/8519425fce7b494c99d331d8cf0a859e, entries=150, sequenceid=308, filesize=12.0 K 2024-11-20T06:40:50,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/a5d35443253f4138932ba1d7115028f9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/a5d35443253f4138932ba1d7115028f9 2024-11-20T06:40:50,783 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/a5d35443253f4138932ba1d7115028f9, entries=150, sequenceid=308, filesize=12.0 K 2024-11-20T06:40:50,784 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=13.42 KB/13740 for d735f3ed3d64a5ca97cafea9e7136d03 in 1693ms, sequenceid=308, compaction requested=false 2024-11-20T06:40:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for d735f3ed3d64a5ca97cafea9e7136d03: 2024-11-20T06:40:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-20T06:40:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-20T06:40:50,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-20T06:40:50,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3770 sec 2024-11-20T06:40:50,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 3.3810 sec 2024-11-20T06:40:51,376 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-11-20T06:40:51,376 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:51,376 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:51,376 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:51,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:51,385 INFO [Thread-678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-678. 2024-11-20T06:40:51,385 DEBUG [Thread-678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:51,385 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:51,385 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:51,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:51,396 INFO [Thread-676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-676. 2024-11-20T06:40:51,396 DEBUG [Thread-676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:40:51,396 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:51,396 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:51,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T06:40:51,584 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3360 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3426 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1522 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4566 rows 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1515 2024-11-20T06:40:51,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4544 rows 2024-11-20T06:40:51,585 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:40:51,585 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:40:51,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:51,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:51,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:40:51,586 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T06:40:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T06:40:51,592 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084851591"}]},"ts":"1732084851591"} 2024-11-20T06:40:51,594 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T06:40:51,594 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T06:40:51,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T06:40:51,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, UNASSIGN}] 2024-11-20T06:40:51,597 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, UNASSIGN 2024-11-20T06:40:51,597 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=d735f3ed3d64a5ca97cafea9e7136d03, regionState=CLOSING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:51,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, UNASSIGN because future has completed 2024-11-20T06:40:51,599 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T06:40:51,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:40:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T06:40:51,751 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:51,751 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T06:40:51,751 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing d735f3ed3d64a5ca97cafea9e7136d03, disabling compactions & flushes 2024-11-20T06:40:51,751 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:51,751 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:51,751 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. after waiting 0 ms 2024-11-20T06:40:51,751 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:51,752 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing d735f3ed3d64a5ca97cafea9e7136d03 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T06:40:51,759 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ea941f69225c4280aec08d3b377a59a1_d735f3ed3d64a5ca97cafea9e7136d03 is 50, key is test_row_0/A:col10/1732084851374/Put/seqid=0 2024-11-20T06:40:51,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742044_1220 (size=12454) 2024-11-20T06:40:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T06:40:52,163 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:52,168 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ea941f69225c4280aec08d3b377a59a1_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ea941f69225c4280aec08d3b377a59a1_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:52,169 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/ecb80667324e425fa117d8e44a027457, store: [table=TestAcidGuarantees family=A region=d735f3ed3d64a5ca97cafea9e7136d03] 2024-11-20T06:40:52,169 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/ecb80667324e425fa117d8e44a027457 is 175, key is test_row_0/A:col10/1732084851374/Put/seqid=0 2024-11-20T06:40:52,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742045_1221 (size=31255) 2024-11-20T06:40:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T06:40:52,574 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/ecb80667324e425fa117d8e44a027457 2024-11-20T06:40:52,583 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/777b9c5ed8474f7eb9264df9b083c6b0 is 50, key is test_row_0/B:col10/1732084851374/Put/seqid=0 2024-11-20T06:40:52,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742046_1222 (size=12301) 2024-11-20T06:40:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T06:40:52,878 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T06:40:52,988 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/777b9c5ed8474f7eb9264df9b083c6b0 2024-11-20T06:40:52,997 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/aefd77c413b442329ce888c72cf2990a is 50, key is test_row_0/C:col10/1732084851374/Put/seqid=0 2024-11-20T06:40:53,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742047_1223 (size=12301) 2024-11-20T06:40:53,402 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/aefd77c413b442329ce888c72cf2990a 2024-11-20T06:40:53,407 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/A/ecb80667324e425fa117d8e44a027457 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ecb80667324e425fa117d8e44a027457 2024-11-20T06:40:53,411 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ecb80667324e425fa117d8e44a027457, entries=150, sequenceid=319, filesize=30.5 K 2024-11-20T06:40:53,412 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/B/777b9c5ed8474f7eb9264df9b083c6b0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/777b9c5ed8474f7eb9264df9b083c6b0 2024-11-20T06:40:53,417 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/777b9c5ed8474f7eb9264df9b083c6b0, entries=150, sequenceid=319, filesize=12.0 K 2024-11-20T06:40:53,418 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/.tmp/C/aefd77c413b442329ce888c72cf2990a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/aefd77c413b442329ce888c72cf2990a 2024-11-20T06:40:53,422 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/aefd77c413b442329ce888c72cf2990a, entries=150, sequenceid=319, filesize=12.0 K 2024-11-20T06:40:53,423 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d735f3ed3d64a5ca97cafea9e7136d03 in 1671ms, sequenceid=319, compaction requested=true 2024-11-20T06:40:53,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/375d74f5ef20485eb48b4041bd6139fe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/2da9714fc0e74548b561dc5136f827c5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/658e197a1fed4c40b8c2b3848bfed465, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/3c4a52d940434939841ccc3da9fc0026, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/a2776a1801074014a599ad0bdbc27b94, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/c926402bf75e4da8a04211015ce2d0de, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/45c4ca627c0b4cd8b0b3579eef684bb2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/75b7dfce836541b1901f25dfe4043a15, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/f157dfeffbf9407e824fb41afd6b63c4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1140ea364d6242a5814a1199a2455320, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6e70ffecebf34328a0547bba9e44e29d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ed2256d0945847a198cb1bc5b0283b4a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/8791820e018c41989e7182515c42973b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/9e167cf0ba2f49cfa0f1ce9a902225ea, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6b9ef52dbfb842d98191033a5f5c52a3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/e5cd654adf2f4fc7884156996431b197, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/64cc5fd704fb4f0288848198881f90dd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/b27d9849bb404a2bba571f8e0af3f7ea, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/7e2e193537274a60817fb0ebc91b7427] to archive 2024-11-20T06:40:53,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:40:53,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/375d74f5ef20485eb48b4041bd6139fe to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/375d74f5ef20485eb48b4041bd6139fe 2024-11-20T06:40:53,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/2da9714fc0e74548b561dc5136f827c5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/2da9714fc0e74548b561dc5136f827c5 2024-11-20T06:40:53,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/658e197a1fed4c40b8c2b3848bfed465 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/658e197a1fed4c40b8c2b3848bfed465 2024-11-20T06:40:53,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/3c4a52d940434939841ccc3da9fc0026 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/3c4a52d940434939841ccc3da9fc0026 2024-11-20T06:40:53,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/a2776a1801074014a599ad0bdbc27b94 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/a2776a1801074014a599ad0bdbc27b94 2024-11-20T06:40:53,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/c926402bf75e4da8a04211015ce2d0de to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/c926402bf75e4da8a04211015ce2d0de 2024-11-20T06:40:53,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/45c4ca627c0b4cd8b0b3579eef684bb2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/45c4ca627c0b4cd8b0b3579eef684bb2 2024-11-20T06:40:53,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/75b7dfce836541b1901f25dfe4043a15 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/75b7dfce836541b1901f25dfe4043a15 2024-11-20T06:40:53,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/f157dfeffbf9407e824fb41afd6b63c4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/f157dfeffbf9407e824fb41afd6b63c4 2024-11-20T06:40:53,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1140ea364d6242a5814a1199a2455320 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1140ea364d6242a5814a1199a2455320 2024-11-20T06:40:53,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6e70ffecebf34328a0547bba9e44e29d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6e70ffecebf34328a0547bba9e44e29d 2024-11-20T06:40:53,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ed2256d0945847a198cb1bc5b0283b4a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ed2256d0945847a198cb1bc5b0283b4a 2024-11-20T06:40:53,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/8791820e018c41989e7182515c42973b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/8791820e018c41989e7182515c42973b 2024-11-20T06:40:53,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/9e167cf0ba2f49cfa0f1ce9a902225ea to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/9e167cf0ba2f49cfa0f1ce9a902225ea 2024-11-20T06:40:53,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6b9ef52dbfb842d98191033a5f5c52a3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/6b9ef52dbfb842d98191033a5f5c52a3 2024-11-20T06:40:53,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/e5cd654adf2f4fc7884156996431b197 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/e5cd654adf2f4fc7884156996431b197 2024-11-20T06:40:53,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/64cc5fd704fb4f0288848198881f90dd to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/64cc5fd704fb4f0288848198881f90dd 2024-11-20T06:40:53,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/b27d9849bb404a2bba571f8e0af3f7ea to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/b27d9849bb404a2bba571f8e0af3f7ea 2024-11-20T06:40:53,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/7e2e193537274a60817fb0ebc91b7427 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/7e2e193537274a60817fb0ebc91b7427 2024-11-20T06:40:53,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/dbb73f56e3a64557888a4292728748f5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/2d41fa7aa5c54d95b6f9982f83f4123c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/cf222933bb4647f6a7e872632b753d3f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/119061bca67c4976aadc9c861e02f6f0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/df565b904a62496d9c0895937a75d51f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/c55ec5c8df4c443ca3eacab491b63aca, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b22af5f28e144eefae793ff4236bf785, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/6cf1dffc86764fc49cdf95448de29847, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/111f36f47da847a3beb55e80f7da61b4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/54b24fe7708d4c68a67cb3c49fc87c2c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a14dc13de642443babb894a97a375e51, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b80cdea77d884f198b8d29bc177cf631, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b35bc11ad7004a14a658f6d420565540, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/704e61f46ac6454ca8f74cd7a60a8720, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/249c01d44b5a4f43b652288e0ad2115e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/db89f050759e4256b5bf2ffdd8ab0e85, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/d8f1ef522062437fbef4b5dd08e4ba55, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/8783209583ac4a9f9d004312b6037a21, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a0da46f1169f4724bc53991e0a649200] to archive 2024-11-20T06:40:53,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:40:53,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/dbb73f56e3a64557888a4292728748f5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/dbb73f56e3a64557888a4292728748f5 2024-11-20T06:40:53,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/2d41fa7aa5c54d95b6f9982f83f4123c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/2d41fa7aa5c54d95b6f9982f83f4123c 2024-11-20T06:40:53,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/cf222933bb4647f6a7e872632b753d3f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/cf222933bb4647f6a7e872632b753d3f 2024-11-20T06:40:53,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/119061bca67c4976aadc9c861e02f6f0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/119061bca67c4976aadc9c861e02f6f0 2024-11-20T06:40:53,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/df565b904a62496d9c0895937a75d51f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/df565b904a62496d9c0895937a75d51f 2024-11-20T06:40:53,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/c55ec5c8df4c443ca3eacab491b63aca to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/c55ec5c8df4c443ca3eacab491b63aca 2024-11-20T06:40:53,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b22af5f28e144eefae793ff4236bf785 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b22af5f28e144eefae793ff4236bf785 2024-11-20T06:40:53,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/6cf1dffc86764fc49cdf95448de29847 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/6cf1dffc86764fc49cdf95448de29847 2024-11-20T06:40:53,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/111f36f47da847a3beb55e80f7da61b4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/111f36f47da847a3beb55e80f7da61b4 2024-11-20T06:40:53,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/54b24fe7708d4c68a67cb3c49fc87c2c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/54b24fe7708d4c68a67cb3c49fc87c2c 2024-11-20T06:40:53,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a14dc13de642443babb894a97a375e51 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a14dc13de642443babb894a97a375e51 2024-11-20T06:40:53,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b80cdea77d884f198b8d29bc177cf631 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b80cdea77d884f198b8d29bc177cf631 2024-11-20T06:40:53,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b35bc11ad7004a14a658f6d420565540 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/b35bc11ad7004a14a658f6d420565540 2024-11-20T06:40:53,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/704e61f46ac6454ca8f74cd7a60a8720 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/704e61f46ac6454ca8f74cd7a60a8720 2024-11-20T06:40:53,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/249c01d44b5a4f43b652288e0ad2115e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/249c01d44b5a4f43b652288e0ad2115e 2024-11-20T06:40:53,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/db89f050759e4256b5bf2ffdd8ab0e85 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/db89f050759e4256b5bf2ffdd8ab0e85 2024-11-20T06:40:53,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/d8f1ef522062437fbef4b5dd08e4ba55 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/d8f1ef522062437fbef4b5dd08e4ba55 2024-11-20T06:40:53,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/8783209583ac4a9f9d004312b6037a21 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/8783209583ac4a9f9d004312b6037a21 2024-11-20T06:40:53,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a0da46f1169f4724bc53991e0a649200 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a0da46f1169f4724bc53991e0a649200 2024-11-20T06:40:53,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/e467d068e689449db9f6670187f431f3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/1392e10989d04132b075ba54f7733559, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/f216cee46a5d420e9e38e654920057f9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/9cb1c65787554f5f8898bb3d2e0f69b1, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/074830ea7c474949acda70998efaaca4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ab9cb831977947cdaa15b511734a0557, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/940d5aee2e124a10afe665308ef4322a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ff1dd14e31144eb3a0f69d5d77dd5e38, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/d161b34b91fc47d2bfff1b517b977233, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/6d0b722d8c6d49eb84faf6ea05937964, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/397cd89fb0424623b3eff8ac2bd728ae, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/4230eb8f129e48a2882dd451a71b9942, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/c66426fcb8104c9bb28da55f930e98c6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/568c20e5e8dd464798601fba7f45fdfe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/b900f4c4d16b4127a5e904c03068cea5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/48b87c4548764ac4b565285599f8417d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/72ba3d8755404019a476f5feea363c4e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/5ca0c8ba49314373ab8b2619cd332c98, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/51eb61f989a64ce0b77402abcae4b09a] to archive 2024-11-20T06:40:53,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:40:53,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/e467d068e689449db9f6670187f431f3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/e467d068e689449db9f6670187f431f3 2024-11-20T06:40:53,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/1392e10989d04132b075ba54f7733559 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/1392e10989d04132b075ba54f7733559 2024-11-20T06:40:53,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/f216cee46a5d420e9e38e654920057f9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/f216cee46a5d420e9e38e654920057f9 2024-11-20T06:40:53,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/9cb1c65787554f5f8898bb3d2e0f69b1 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/9cb1c65787554f5f8898bb3d2e0f69b1 2024-11-20T06:40:53,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/074830ea7c474949acda70998efaaca4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/074830ea7c474949acda70998efaaca4 2024-11-20T06:40:53,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ab9cb831977947cdaa15b511734a0557 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ab9cb831977947cdaa15b511734a0557 2024-11-20T06:40:53,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/940d5aee2e124a10afe665308ef4322a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/940d5aee2e124a10afe665308ef4322a 2024-11-20T06:40:53,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ff1dd14e31144eb3a0f69d5d77dd5e38 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/ff1dd14e31144eb3a0f69d5d77dd5e38 2024-11-20T06:40:53,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/d161b34b91fc47d2bfff1b517b977233 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/d161b34b91fc47d2bfff1b517b977233 2024-11-20T06:40:53,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/6d0b722d8c6d49eb84faf6ea05937964 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/6d0b722d8c6d49eb84faf6ea05937964 2024-11-20T06:40:53,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/397cd89fb0424623b3eff8ac2bd728ae to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/397cd89fb0424623b3eff8ac2bd728ae 2024-11-20T06:40:53,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/4230eb8f129e48a2882dd451a71b9942 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/4230eb8f129e48a2882dd451a71b9942 2024-11-20T06:40:53,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/c66426fcb8104c9bb28da55f930e98c6 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/c66426fcb8104c9bb28da55f930e98c6 2024-11-20T06:40:53,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/568c20e5e8dd464798601fba7f45fdfe to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/568c20e5e8dd464798601fba7f45fdfe 2024-11-20T06:40:53,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/b900f4c4d16b4127a5e904c03068cea5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/b900f4c4d16b4127a5e904c03068cea5 2024-11-20T06:40:53,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/48b87c4548764ac4b565285599f8417d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/48b87c4548764ac4b565285599f8417d 2024-11-20T06:40:53,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/72ba3d8755404019a476f5feea363c4e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/72ba3d8755404019a476f5feea363c4e 2024-11-20T06:40:53,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/5ca0c8ba49314373ab8b2619cd332c98 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/5ca0c8ba49314373ab8b2619cd332c98 2024-11-20T06:40:53,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/51eb61f989a64ce0b77402abcae4b09a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/51eb61f989a64ce0b77402abcae4b09a 2024-11-20T06:40:53,523 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/recovered.edits/322.seqid, newMaxSeqId=322, maxSeqId=4 2024-11-20T06:40:53,524 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. 2024-11-20T06:40:53,524 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for d735f3ed3d64a5ca97cafea9e7136d03: Waiting for close lock at 1732084851751Running coprocessor pre-close hooks at 1732084851751Disabling compacts and flushes for region at 1732084851751Disabling writes for close at 1732084851751Obtaining lock to block concurrent updates at 1732084851752 (+1 ms)Preparing flush snapshotting stores in d735f3ed3d64a5ca97cafea9e7136d03 at 1732084851752Finished memstore snapshotting TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732084851752Flushing stores of TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03. at 1732084851752Flushing d735f3ed3d64a5ca97cafea9e7136d03/A: creating writer at 1732084851752Flushing mob file d735f3ed3d64a5ca97cafea9e7136d03/A: appending metadata at 1732084851758 (+6 ms)Flushing mob file d735f3ed3d64a5ca97cafea9e7136d03/A: closing flushed file at 1732084851758Flushing d735f3ed3d64a5ca97cafea9e7136d03/A: appending metadata at 1732084852169 (+411 ms)Flushing d735f3ed3d64a5ca97cafea9e7136d03/A: closing flushed file at 1732084852169Flushing d735f3ed3d64a5ca97cafea9e7136d03/B: creating writer at 1732084852580 (+411 ms)Flushing d735f3ed3d64a5ca97cafea9e7136d03/B: appending metadata at 1732084852583 (+3 ms)Flushing d735f3ed3d64a5ca97cafea9e7136d03/B: closing flushed file at 1732084852583Flushing d735f3ed3d64a5ca97cafea9e7136d03/C: creating writer at 1732084852993 (+410 ms)Flushing d735f3ed3d64a5ca97cafea9e7136d03/C: appending metadata at 1732084852996 (+3 ms)Flushing d735f3ed3d64a5ca97cafea9e7136d03/C: closing flushed file at 1732084852996Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@729b8c22: reopening flushed file at 1732084853406 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2efcf063: reopening flushed file at 1732084853411 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ccd537f: reopening flushed file at 1732084853417 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d735f3ed3d64a5ca97cafea9e7136d03 in 1671ms, sequenceid=319, compaction requested=true at 1732084853423 (+6 ms)Writing region close event to WAL at 1732084853519 (+96 ms)Running coprocessor post-close hooks at 1732084853524 (+5 ms)Closed at 1732084853524 2024-11-20T06:40:53,527 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,527 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=d735f3ed3d64a5ca97cafea9e7136d03, regionState=CLOSED 2024-11-20T06:40:53,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:40:53,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T06:40:53,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure d735f3ed3d64a5ca97cafea9e7136d03, server=61c2066a8a0f,33847,1732084794617 in 1.9300 sec 2024-11-20T06:40:53,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-20T06:40:53,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d735f3ed3d64a5ca97cafea9e7136d03, UNASSIGN in 1.9350 sec 2024-11-20T06:40:53,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-20T06:40:53,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9380 sec 2024-11-20T06:40:53,536 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084853536"}]},"ts":"1732084853536"} 2024-11-20T06:40:53,538 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T06:40:53,538 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T06:40:53,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9520 sec 2024-11-20T06:40:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T06:40:53,734 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:53,735 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T06:40:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:53,736 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T06:40:53,737 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:53,740 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,744 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/recovered.edits] 2024-11-20T06:40:53,749 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1110e4877b13434191dfd4f14ee4c8e0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/1110e4877b13434191dfd4f14ee4c8e0 2024-11-20T06:40:53,751 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/aa7f56991b1a4f4a9a8071667f8ddb3e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/aa7f56991b1a4f4a9a8071667f8ddb3e 2024-11-20T06:40:53,752 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ecb80667324e425fa117d8e44a027457 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/A/ecb80667324e425fa117d8e44a027457 2024-11-20T06:40:53,755 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/777b9c5ed8474f7eb9264df9b083c6b0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/777b9c5ed8474f7eb9264df9b083c6b0 2024-11-20T06:40:53,757 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/8519425fce7b494c99d331d8cf0a859e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/8519425fce7b494c99d331d8cf0a859e 2024-11-20T06:40:53,758 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a09317b2b76d4f9ea8bd1d1136cabf62 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/B/a09317b2b76d4f9ea8bd1d1136cabf62 2024-11-20T06:40:53,761 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/a5d35443253f4138932ba1d7115028f9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/a5d35443253f4138932ba1d7115028f9 2024-11-20T06:40:53,762 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/aefd77c413b442329ce888c72cf2990a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/aefd77c413b442329ce888c72cf2990a 2024-11-20T06:40:53,763 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/de7f8fd21c2d46b1a45017af13f91005 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/C/de7f8fd21c2d46b1a45017af13f91005 2024-11-20T06:40:53,768 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/recovered.edits/322.seqid to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03/recovered.edits/322.seqid 2024-11-20T06:40:53,768 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,769 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T06:40:53,769 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T06:40:53,770 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T06:40:53,775 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201535636b822d4eb1a34ba1d0062e562b_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201535636b822d4eb1a34ba1d0062e562b_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,776 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112015adfd8c0ca6417189f0a60ec5aac8e0_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112015adfd8c0ca6417189f0a60ec5aac8e0_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,777 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120210716ad9ac24f75b5e552a6a6581b3e_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120210716ad9ac24f75b5e552a6a6581b3e_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,778 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112025b04a8a12cf42638ce09d0d7ad43632_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112025b04a8a12cf42638ce09d0d7ad43632_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,779 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202621ea51342e4ee287aa095977f371bd_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202621ea51342e4ee287aa095977f371bd_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,780 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203b0c4ffb76a241b08b689d9be973ea6f_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203b0c4ffb76a241b08b689d9be973ea6f_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,781 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203e4536415c384b649d3a48db752a8bf8_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203e4536415c384b649d3a48db752a8bf8_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,782 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205a43b49e12214a718248a6a0c4cdbf99_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205a43b49e12214a718248a6a0c4cdbf99_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,783 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206668c9429f344344ab8ae283004943e8_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206668c9429f344344ab8ae283004943e8_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,784 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120727fa60674cc4b8c99ca0bcd1713bf71_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120727fa60674cc4b8c99ca0bcd1713bf71_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,785 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120baa6f621c0954abe8f27af331b6a744d_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120baa6f621c0954abe8f27af331b6a744d_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,787 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be23ebd62eaa4387b8ec29a8004a65cf_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be23ebd62eaa4387b8ec29a8004a65cf_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,788 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cb0ae28328584b2bbf71d9023f85a4d8_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cb0ae28328584b2bbf71d9023f85a4d8_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,790 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cee06e7a8de840d9b1a07de88a5599aa_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cee06e7a8de840d9b1a07de88a5599aa_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,791 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de655e0e92f24e0cb47e984aabe71d42_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120de655e0e92f24e0cb47e984aabe71d42_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,792 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ea941f69225c4280aec08d3b377a59a1_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ea941f69225c4280aec08d3b377a59a1_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,793 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f83cdaa44e9145d785d24b179ad32f30_d735f3ed3d64a5ca97cafea9e7136d03 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f83cdaa44e9145d785d24b179ad32f30_d735f3ed3d64a5ca97cafea9e7136d03 2024-11-20T06:40:53,794 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T06:40:53,796 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:53,799 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T06:40:53,801 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T06:40:53,803 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:53,803 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T06:40:53,803 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732084853803"}]},"ts":"9223372036854775807"} 2024-11-20T06:40:53,807 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T06:40:53,807 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d735f3ed3d64a5ca97cafea9e7136d03, NAME => 'TestAcidGuarantees,,1732084826111.d735f3ed3d64a5ca97cafea9e7136d03.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T06:40:53,807 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T06:40:53,807 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732084853807"}]},"ts":"9223372036854775807"} 2024-11-20T06:40:53,810 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T06:40:53,811 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:53,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 76 msec 2024-11-20T06:40:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T06:40:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T06:40:53,844 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:53,854 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240 (was 238) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/cluster_a8a03e82-e5a9-0028-3abc-1ed331460361/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1351162969_22 at /127.0.0.1:53162 [Waiting for operation #588] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1532015227_22 at /127.0.0.1:48380 [Waiting for operation #589] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/cluster_a8a03e82-e5a9-0028-3abc-1ed331460361/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=467 (was 459) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4041 (was 5125) 2024-11-20T06:40:53,866 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=467, ProcessCount=11, AvailableMemoryMB=4040 2024-11-20T06:40:53,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T06:40:53,867 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:40:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T06:40:53,869 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T06:40:53,869 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:53,869 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-20T06:40:53,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T06:40:53,870 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T06:40:53,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742048_1224 (size=959) 2024-11-20T06:40:53,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T06:40:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T06:40:54,221 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T06:40:54,278 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5 2024-11-20T06:40:54,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742049_1225 (size=53) 2024-11-20T06:40:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T06:40:54,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:40:54,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 792921c2784cd0f6a5ba47b7f4462f5c, disabling compactions & flushes 2024-11-20T06:40:54,686 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:54,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:54,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. after waiting 0 ms 2024-11-20T06:40:54,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:54,686 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:54,686 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 792921c2784cd0f6a5ba47b7f4462f5c: Waiting for close lock at 1732084854686Disabling compacts and flushes for region at 1732084854686Disabling writes for close at 1732084854686Writing region close event to WAL at 1732084854686Closed at 1732084854686 2024-11-20T06:40:54,689 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T06:40:54,690 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732084854689"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732084854689"}]},"ts":"1732084854689"} 2024-11-20T06:40:54,693 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T06:40:54,694 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T06:40:54,695 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084854694"}]},"ts":"1732084854694"} 2024-11-20T06:40:54,698 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T06:40:54,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=792921c2784cd0f6a5ba47b7f4462f5c, ASSIGN}] 2024-11-20T06:40:54,699 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=792921c2784cd0f6a5ba47b7f4462f5c, ASSIGN 2024-11-20T06:40:54,700 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=792921c2784cd0f6a5ba47b7f4462f5c, ASSIGN; state=OFFLINE, location=61c2066a8a0f,33847,1732084794617; forceNewPlan=false, retain=false 2024-11-20T06:40:54,851 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=792921c2784cd0f6a5ba47b7f4462f5c, regionState=OPENING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:54,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=792921c2784cd0f6a5ba47b7f4462f5c, ASSIGN because future has completed 2024-11-20T06:40:54,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:40:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T06:40:55,008 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:55,009 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:40:55,009 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,009 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:40:55,009 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,009 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,015 INFO [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,016 INFO [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 792921c2784cd0f6a5ba47b7f4462f5c columnFamilyName A 2024-11-20T06:40:55,016 DEBUG [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:55,017 INFO [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] regionserver.HStore(327): Store=792921c2784cd0f6a5ba47b7f4462f5c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:55,017 INFO [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,019 INFO [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 792921c2784cd0f6a5ba47b7f4462f5c columnFamilyName B 2024-11-20T06:40:55,019 DEBUG [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:55,019 INFO [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] regionserver.HStore(327): Store=792921c2784cd0f6a5ba47b7f4462f5c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:55,019 INFO [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,021 INFO [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 792921c2784cd0f6a5ba47b7f4462f5c columnFamilyName C 2024-11-20T06:40:55,021 DEBUG [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:40:55,021 INFO [StoreOpener-792921c2784cd0f6a5ba47b7f4462f5c-1 {}] regionserver.HStore(327): Store=792921c2784cd0f6a5ba47b7f4462f5c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:40:55,021 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,022 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,022 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,023 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,023 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,023 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:40:55,024 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,026 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:40:55,027 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 792921c2784cd0f6a5ba47b7f4462f5c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60780435, jitterRate=-0.09430094063282013}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:40:55,027 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:55,027 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 792921c2784cd0f6a5ba47b7f4462f5c: Running coprocessor pre-open hook at 1732084855009Writing region info on filesystem at 1732084855009Initializing all the Stores at 1732084855010 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084855010Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084855014 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084855014Cleaning up temporary data from old regions at 1732084855023 (+9 ms)Running coprocessor post-open hooks at 1732084855027 (+4 ms)Region opened successfully at 1732084855027 2024-11-20T06:40:55,028 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., pid=65, masterSystemTime=1732084855005 2024-11-20T06:40:55,032 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:55,032 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:55,033 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=792921c2784cd0f6a5ba47b7f4462f5c, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:55,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:40:55,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-20T06:40:55,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 in 182 msec 2024-11-20T06:40:55,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-20T06:40:55,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=792921c2784cd0f6a5ba47b7f4462f5c, ASSIGN in 339 msec 2024-11-20T06:40:55,039 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T06:40:55,040 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084855040"}]},"ts":"1732084855040"} 2024-11-20T06:40:55,041 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T06:40:55,042 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T06:40:55,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-20T06:40:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-20T06:40:56,016 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:40:56,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5832a527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b01fb86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eea9e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,027 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,030 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T06:40:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,33659,1732084793752 2024-11-20T06:40:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1332a5a4 2024-11-20T06:40:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T06:40:56,032 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T06:40:56,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a08a8a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,034 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34c353bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,035 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56a145ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ddfbea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,037 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6f1727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,038 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f082976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fbbb442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,040 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,040 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df5e5f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,042 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b69fbf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341051a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e72683d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,045 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e861a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309aa0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,047 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494a21a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,049 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219f3332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b624df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,051 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58993301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,052 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f86561, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106f6d84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,054 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300e21e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,056 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56735fd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a00a41f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,057 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef1b54a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,059 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e26b1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1b1c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,061 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78bbcfa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,062 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f269a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@727224d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,063 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:40:56,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:40:56,064 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:40:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:40:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:40:56,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d56c51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:40:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:40:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:40:56,065 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:40:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ec9d0d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:40:56,066 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,066 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,067 DEBUG [Thread-1062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,067 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:40:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,067 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,067 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,068 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,068 DEBUG [Thread-1067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-20T06:40:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T06:40:56,070 DEBUG [Thread-1067 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,070 DEBUG [Thread-1067 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:56,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,071 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:40:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:56,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:56,071 DEBUG [Thread-1065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:56,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,073 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:40:56,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:40:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,074 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,075 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:40:56,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:56,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:56,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:56,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:40:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:40:56,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:56,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:40:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:40:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:56,090 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:56,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T06:40:56,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/f825710b29e042648a9b2ed7151344d9 is 50, key is test_row_0/A:col10/1732084856089/Put/seqid=0 2024-11-20T06:40:56,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084916113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084916111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084916120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084916120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084916120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742050_1226 (size=16681) 2024-11-20T06:40:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T06:40:56,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T06:40:56,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:40:56,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084916234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084916234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084916234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084916235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084916235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T06:40:56,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:40:56,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T06:40:56,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084916443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084916443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084916444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084916444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084916444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T06:40:56,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:40:56,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/f825710b29e042648a9b2ed7151344d9 2024-11-20T06:40:56,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/581cb75591924e57810221793a8b6b51 is 50, key is test_row_0/B:col10/1732084856089/Put/seqid=0 2024-11-20T06:40:56,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742051_1227 (size=12001) 2024-11-20T06:40:56,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T06:40:56,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:40:56,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T06:40:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084916754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084916754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084916754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084916754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084916754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:56,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:56,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T06:40:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:40:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/581cb75591924e57810221793a8b6b51 2024-11-20T06:40:56,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T06:40:56,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:40:56,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:56,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:57,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/775382e17f774efd894dc2744e4eae29 is 50, key is test_row_0/C:col10/1732084856089/Put/seqid=0 2024-11-20T06:40:57,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742052_1228 (size=12001) 2024-11-20T06:40:57,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T06:40:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:40:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:57,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T06:40:57,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084917264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:57,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:57,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084917264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:57,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:57,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:57,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084917264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:57,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084917264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:57,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084917264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:57,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T06:40:57,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:40:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:57,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:57,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/775382e17f774efd894dc2744e4eae29 2024-11-20T06:40:57,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/f825710b29e042648a9b2ed7151344d9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f825710b29e042648a9b2ed7151344d9 2024-11-20T06:40:57,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f825710b29e042648a9b2ed7151344d9, entries=250, sequenceid=17, filesize=16.3 K 2024-11-20T06:40:57,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/581cb75591924e57810221793a8b6b51 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/581cb75591924e57810221793a8b6b51 2024-11-20T06:40:57,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/581cb75591924e57810221793a8b6b51, entries=150, sequenceid=17, filesize=11.7 K 2024-11-20T06:40:57,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/775382e17f774efd894dc2744e4eae29 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/775382e17f774efd894dc2744e4eae29 2024-11-20T06:40:57,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T06:40:57,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/775382e17f774efd894dc2744e4eae29, entries=150, sequenceid=17, filesize=11.7 K 2024-11-20T06:40:57,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:57,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:40:57,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:57,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:57,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:40:57,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 792921c2784cd0f6a5ba47b7f4462f5c in 1377ms, sequenceid=17, compaction requested=false 2024-11-20T06:40:57,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:40:57,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T06:40:57,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:57,618 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T06:40:57,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/bdb21a5513a14d769f6e5414dfa69ffb is 50, key is test_row_0/A:col10/1732084856116/Put/seqid=0 2024-11-20T06:40:57,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742053_1229 (size=12001) 2024-11-20T06:40:58,027 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/bdb21a5513a14d769f6e5414dfa69ffb 2024-11-20T06:40:58,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/b4e172ce28d34131878642326cdbdb35 is 50, key is test_row_0/B:col10/1732084856116/Put/seqid=0 2024-11-20T06:40:58,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742054_1230 (size=12001) 2024-11-20T06:40:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T06:40:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:58,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:40:58,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084918288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084918289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084918289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084918290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084918290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084918404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084918404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084918404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084918404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084918404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,450 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/b4e172ce28d34131878642326cdbdb35 2024-11-20T06:40:58,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/bc6169fe33464d3cb4b7fa8b1df1c82c is 50, key is test_row_0/C:col10/1732084856116/Put/seqid=0 2024-11-20T06:40:58,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742055_1231 (size=12001) 2024-11-20T06:40:58,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084918614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084918614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084918614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084918614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084918614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,862 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/bc6169fe33464d3cb4b7fa8b1df1c82c 2024-11-20T06:40:58,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/bdb21a5513a14d769f6e5414dfa69ffb as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bdb21a5513a14d769f6e5414dfa69ffb 2024-11-20T06:40:58,870 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bdb21a5513a14d769f6e5414dfa69ffb, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T06:40:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/b4e172ce28d34131878642326cdbdb35 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b4e172ce28d34131878642326cdbdb35 2024-11-20T06:40:58,875 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b4e172ce28d34131878642326cdbdb35, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T06:40:58,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/bc6169fe33464d3cb4b7fa8b1df1c82c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/bc6169fe33464d3cb4b7fa8b1df1c82c 2024-11-20T06:40:58,879 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/bc6169fe33464d3cb4b7fa8b1df1c82c, entries=150, sequenceid=38, filesize=11.7 K 2024-11-20T06:40:58,880 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 792921c2784cd0f6a5ba47b7f4462f5c in 1262ms, sequenceid=38, compaction requested=false 2024-11-20T06:40:58,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:40:58,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:40:58,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-20T06:40:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-20T06:40:58,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-20T06:40:58,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8090 sec 2024-11-20T06:40:58,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.8160 sec 2024-11-20T06:40:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:40:58,926 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:40:58,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T06:40:58,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c1119381facb40c29ccc40fd8512ca4d is 50, key is test_row_0/A:col10/1732084858925/Put/seqid=0 2024-11-20T06:40:58,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742056_1232 (size=12001) 2024-11-20T06:40:58,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084918939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084918940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084918942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084918941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084918944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,050 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T06:40:59,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084919054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084919054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084919054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084919054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084919054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084919264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084919264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084919264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084919265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084919265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c1119381facb40c29ccc40fd8512ca4d 2024-11-20T06:40:59,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/0631f4c00fbd4e2b880b5568422a679e is 50, key is test_row_0/B:col10/1732084858925/Put/seqid=0 2024-11-20T06:40:59,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742057_1233 (size=12001) 2024-11-20T06:40:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084919574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084919574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084919574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084919574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:40:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084919574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:40:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:40:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:40:59,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/0631f4c00fbd4e2b880b5568422a679e 2024-11-20T06:40:59,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/15636d6b9ba64690a97b17f7a0980ddf is 50, key is test_row_0/C:col10/1732084858925/Put/seqid=0 2024-11-20T06:40:59,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742058_1234 (size=12001) 2024-11-20T06:41:00,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084920083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:00,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084920083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:00,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084920084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:00,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084920084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:00,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084920094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:00,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/15636d6b9ba64690a97b17f7a0980ddf 2024-11-20T06:41:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c1119381facb40c29ccc40fd8512ca4d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c1119381facb40c29ccc40fd8512ca4d 2024-11-20T06:41:00,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c1119381facb40c29ccc40fd8512ca4d, entries=150, sequenceid=56, filesize=11.7 K 2024-11-20T06:41:00,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/0631f4c00fbd4e2b880b5568422a679e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0631f4c00fbd4e2b880b5568422a679e 2024-11-20T06:41:00,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0631f4c00fbd4e2b880b5568422a679e, entries=150, sequenceid=56, filesize=11.7 K 2024-11-20T06:41:00,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/15636d6b9ba64690a97b17f7a0980ddf as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/15636d6b9ba64690a97b17f7a0980ddf 2024-11-20T06:41:00,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/15636d6b9ba64690a97b17f7a0980ddf, entries=150, sequenceid=56, filesize=11.7 K 2024-11-20T06:41:00,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 792921c2784cd0f6a5ba47b7f4462f5c in 1263ms, sequenceid=56, compaction requested=true 2024-11-20T06:41:00,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:00,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:00,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:00,190 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:00,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:00,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:00,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:00,190 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:00,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:00,191 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:00,191 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:00,191 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:00,191 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:00,191 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:00,191 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:00,191 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/581cb75591924e57810221793a8b6b51, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b4e172ce28d34131878642326cdbdb35, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0631f4c00fbd4e2b880b5568422a679e] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=35.2 K 2024-11-20T06:41:00,191 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f825710b29e042648a9b2ed7151344d9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bdb21a5513a14d769f6e5414dfa69ffb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c1119381facb40c29ccc40fd8512ca4d] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=39.7 K 2024-11-20T06:41:00,191 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting f825710b29e042648a9b2ed7151344d9, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732084856081 2024-11-20T06:41:00,191 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 581cb75591924e57810221793a8b6b51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732084856081 2024-11-20T06:41:00,192 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdb21a5513a14d769f6e5414dfa69ffb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732084856111 2024-11-20T06:41:00,192 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b4e172ce28d34131878642326cdbdb35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732084856111 2024-11-20T06:41:00,192 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 0631f4c00fbd4e2b880b5568422a679e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732084858925 2024-11-20T06:41:00,192 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1119381facb40c29ccc40fd8512ca4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732084858925 2024-11-20T06:41:00,200 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:00,201 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:00,201 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/a02f9d0e4f5a4e3e8cd3fccc0d8fef2d is 50, key is test_row_0/B:col10/1732084858925/Put/seqid=0 2024-11-20T06:41:00,201 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/5014baa2409f4162b01fa662b3f0bd1f is 50, key is test_row_0/A:col10/1732084858925/Put/seqid=0 2024-11-20T06:41:00,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742059_1235 (size=12104) 2024-11-20T06:41:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T06:41:00,224 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:00,225 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-20T06:41:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T06:41:00,227 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:00,227 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:00,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:00,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742060_1236 (size=12104) 2024-11-20T06:41:00,236 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/5014baa2409f4162b01fa662b3f0bd1f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/5014baa2409f4162b01fa662b3f0bd1f 2024-11-20T06:41:00,242 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into 5014baa2409f4162b01fa662b3f0bd1f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:00,242 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:00,242 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084860190; duration=0sec 2024-11-20T06:41:00,242 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:00,242 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:00,242 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:00,243 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:00,243 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:00,243 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:00,243 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/775382e17f774efd894dc2744e4eae29, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/bc6169fe33464d3cb4b7fa8b1df1c82c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/15636d6b9ba64690a97b17f7a0980ddf] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=35.2 K 2024-11-20T06:41:00,244 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 775382e17f774efd894dc2744e4eae29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732084856081 2024-11-20T06:41:00,244 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc6169fe33464d3cb4b7fa8b1df1c82c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732084856111 2024-11-20T06:41:00,244 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15636d6b9ba64690a97b17f7a0980ddf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732084858925 2024-11-20T06:41:00,252 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:00,253 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/761f3a32bb9c4a039c14218e234d6c3f is 50, key is test_row_0/C:col10/1732084858925/Put/seqid=0 2024-11-20T06:41:00,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742061_1237 (size=12104) 2024-11-20T06:41:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T06:41:00,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T06:41:00,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:00,383 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T06:41:00,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/f5d20f6aff5040dd9b2a73a72df51c3f is 50, key is test_row_0/A:col10/1732084858941/Put/seqid=0 2024-11-20T06:41:00,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742062_1238 (size=12001) 2024-11-20T06:41:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T06:41:00,621 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/a02f9d0e4f5a4e3e8cd3fccc0d8fef2d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a02f9d0e4f5a4e3e8cd3fccc0d8fef2d 2024-11-20T06:41:00,625 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into a02f9d0e4f5a4e3e8cd3fccc0d8fef2d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:00,625 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:00,625 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084860190; duration=0sec 2024-11-20T06:41:00,626 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:00,626 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:00,662 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/761f3a32bb9c4a039c14218e234d6c3f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/761f3a32bb9c4a039c14218e234d6c3f 2024-11-20T06:41:00,667 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into 761f3a32bb9c4a039c14218e234d6c3f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:00,667 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:00,667 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084860190; duration=0sec 2024-11-20T06:41:00,667 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:00,667 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:00,795 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/f5d20f6aff5040dd9b2a73a72df51c3f 2024-11-20T06:41:00,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/085bd307591e4008b26b25f99e46f8fe is 50, key is test_row_0/B:col10/1732084858941/Put/seqid=0 2024-11-20T06:41:00,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742063_1239 (size=12001) 2024-11-20T06:41:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T06:41:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:01,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:01,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084921108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084921109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084921110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084921111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084921112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,206 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/085bd307591e4008b26b25f99e46f8fe 2024-11-20T06:41:01,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/cb59db1744a34523bab9871c249a4ff5 is 50, key is test_row_0/C:col10/1732084858941/Put/seqid=0 2024-11-20T06:41:01,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084921214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742064_1240 (size=12001) 2024-11-20T06:41:01,219 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/cb59db1744a34523bab9871c249a4ff5 2024-11-20T06:41:01,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/f5d20f6aff5040dd9b2a73a72df51c3f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f5d20f6aff5040dd9b2a73a72df51c3f 2024-11-20T06:41:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084921224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084921224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084921224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084921224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,229 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f5d20f6aff5040dd9b2a73a72df51c3f, entries=150, sequenceid=75, filesize=11.7 K 2024-11-20T06:41:01,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/085bd307591e4008b26b25f99e46f8fe as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/085bd307591e4008b26b25f99e46f8fe 2024-11-20T06:41:01,234 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/085bd307591e4008b26b25f99e46f8fe, entries=150, sequenceid=75, filesize=11.7 K 2024-11-20T06:41:01,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/cb59db1744a34523bab9871c249a4ff5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cb59db1744a34523bab9871c249a4ff5 2024-11-20T06:41:01,238 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cb59db1744a34523bab9871c249a4ff5, entries=150, sequenceid=75, filesize=11.7 K 2024-11-20T06:41:01,239 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 792921c2784cd0f6a5ba47b7f4462f5c in 856ms, sequenceid=75, compaction requested=false 2024-11-20T06:41:01,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:01,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:01,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-20T06:41:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-20T06:41:01,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-20T06:41:01,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0130 sec 2024-11-20T06:41:01,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0180 sec 2024-11-20T06:41:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T06:41:01,364 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:01,364 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-20T06:41:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T06:41:01,366 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:01,367 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:01,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:01,424 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:01,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:41:01,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c6eabad16660451ca1bff5aeb2432ae0 is 50, key is test_row_0/A:col10/1732084861110/Put/seqid=0 2024-11-20T06:41:01,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742065_1241 (size=11997) 2024-11-20T06:41:01,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084921438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084921440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084921441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084921441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084921442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T06:41:01,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T06:41:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:01,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084921544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084921554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084921554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084921554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084921554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T06:41:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:01,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T06:41:01,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084921753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084921763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084921763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084921764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084921764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:01,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T06:41:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:01,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c6eabad16660451ca1bff5aeb2432ae0 2024-11-20T06:41:01,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/1243f555e6bb4f46b82ada93332e3ac0 is 50, key is test_row_0/B:col10/1732084861110/Put/seqid=0 2024-11-20T06:41:01,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742066_1242 (size=9657) 2024-11-20T06:41:01,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T06:41:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:01,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T06:41:02,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084922063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084922073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084922074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084922074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:02,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084922074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:02,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T06:41:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/1243f555e6bb4f46b82ada93332e3ac0 2024-11-20T06:41:02,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/9dc77f229e184209acd629d459694213 is 50, key is test_row_0/C:col10/1732084861110/Put/seqid=0 2024-11-20T06:41:02,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742067_1243 (size=9657) 2024-11-20T06:41:02,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T06:41:02,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:02,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T06:41:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:02,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T06:41:02,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084922573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:02,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084922583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:02,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084922584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084922584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:02,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084922584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:02,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T06:41:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:02,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/9dc77f229e184209acd629d459694213 2024-11-20T06:41:02,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c6eabad16660451ca1bff5aeb2432ae0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c6eabad16660451ca1bff5aeb2432ae0 2024-11-20T06:41:02,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c6eabad16660451ca1bff5aeb2432ae0, entries=150, sequenceid=97, filesize=11.7 K 2024-11-20T06:41:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/1243f555e6bb4f46b82ada93332e3ac0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/1243f555e6bb4f46b82ada93332e3ac0 2024-11-20T06:41:02,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/1243f555e6bb4f46b82ada93332e3ac0, entries=100, sequenceid=97, filesize=9.4 K 2024-11-20T06:41:02,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/9dc77f229e184209acd629d459694213 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9dc77f229e184209acd629d459694213 2024-11-20T06:41:02,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9dc77f229e184209acd629d459694213, entries=100, sequenceid=97, filesize=9.4 K 2024-11-20T06:41:02,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 792921c2784cd0f6a5ba47b7f4462f5c in 1275ms, sequenceid=97, compaction requested=true 2024-11-20T06:41:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:02,699 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:02,699 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:02,700 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:02,700 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:02,700 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,701 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a02f9d0e4f5a4e3e8cd3fccc0d8fef2d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/085bd307591e4008b26b25f99e46f8fe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/1243f555e6bb4f46b82ada93332e3ac0] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=33.0 K 2024-11-20T06:41:02,701 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:02,701 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:02,701 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,701 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/5014baa2409f4162b01fa662b3f0bd1f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f5d20f6aff5040dd9b2a73a72df51c3f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c6eabad16660451ca1bff5aeb2432ae0] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=35.3 K 2024-11-20T06:41:02,701 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a02f9d0e4f5a4e3e8cd3fccc0d8fef2d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732084858925 2024-11-20T06:41:02,701 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5014baa2409f4162b01fa662b3f0bd1f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732084858925 2024-11-20T06:41:02,702 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 085bd307591e4008b26b25f99e46f8fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732084858936 2024-11-20T06:41:02,702 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5d20f6aff5040dd9b2a73a72df51c3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732084858936 2024-11-20T06:41:02,702 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 1243f555e6bb4f46b82ada93332e3ac0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732084861110 2024-11-20T06:41:02,702 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6eabad16660451ca1bff5aeb2432ae0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732084861110 2024-11-20T06:41:02,710 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:02,710 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:02,711 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/39de4f93bc56433389a3e2fd67d6542b is 50, key is test_row_0/A:col10/1732084861110/Put/seqid=0 2024-11-20T06:41:02,711 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/d9e47c61b8b14819a4838b03e96ed823 is 50, key is test_row_0/B:col10/1732084861110/Put/seqid=0 2024-11-20T06:41:02,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742068_1244 (size=12207) 2024-11-20T06:41:02,720 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/d9e47c61b8b14819a4838b03e96ed823 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d9e47c61b8b14819a4838b03e96ed823 2024-11-20T06:41:02,726 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into d9e47c61b8b14819a4838b03e96ed823(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:02,726 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:02,726 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084862699; duration=0sec 2024-11-20T06:41:02,726 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:02,726 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:02,726 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:02,728 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:02,728 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:02,728 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,728 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/761f3a32bb9c4a039c14218e234d6c3f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cb59db1744a34523bab9871c249a4ff5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9dc77f229e184209acd629d459694213] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=33.0 K 2024-11-20T06:41:02,728 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 761f3a32bb9c4a039c14218e234d6c3f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732084858925 2024-11-20T06:41:02,728 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting cb59db1744a34523bab9871c249a4ff5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732084858936 2024-11-20T06:41:02,729 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dc77f229e184209acd629d459694213, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732084861110 2024-11-20T06:41:02,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742069_1245 (size=12207) 2024-11-20T06:41:02,737 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/39de4f93bc56433389a3e2fd67d6542b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/39de4f93bc56433389a3e2fd67d6542b 2024-11-20T06:41:02,737 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:02,738 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/e476f1c5f2ad48039c2f531275ee1149 is 50, key is test_row_0/C:col10/1732084861110/Put/seqid=0 2024-11-20T06:41:02,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T06:41:02,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:02,743 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T06:41:02,743 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into 39de4f93bc56433389a3e2fd67d6542b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:02,743 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:02,743 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084862699; duration=0sec 2024-11-20T06:41:02,743 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:02,743 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:02,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742070_1246 (size=12207) 2024-11-20T06:41:02,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/6a84fda3aaa04b1aa7d55630eeddcb39 is 50, key is test_row_0/A:col10/1732084861440/Put/seqid=0 2024-11-20T06:41:02,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742071_1247 (size=12001) 2024-11-20T06:41:02,754 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/e476f1c5f2ad48039c2f531275ee1149 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e476f1c5f2ad48039c2f531275ee1149 2024-11-20T06:41:02,754 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/6a84fda3aaa04b1aa7d55630eeddcb39 2024-11-20T06:41:02,764 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into e476f1c5f2ad48039c2f531275ee1149(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:02,764 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:02,764 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084862699; duration=0sec 2024-11-20T06:41:02,764 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:02,764 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/86d6e1c6f6054c639d078ce81bffbe29 is 50, key is test_row_0/B:col10/1732084861440/Put/seqid=0 2024-11-20T06:41:02,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742072_1248 (size=12001) 2024-11-20T06:41:03,181 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/86d6e1c6f6054c639d078ce81bffbe29 2024-11-20T06:41:03,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/547cb734a78447b5b13e05c08a7116f2 is 50, key is test_row_0/C:col10/1732084861440/Put/seqid=0 2024-11-20T06:41:03,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742073_1249 (size=12001) 2024-11-20T06:41:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T06:41:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:03,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:03,595 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/547cb734a78447b5b13e05c08a7116f2 2024-11-20T06:41:03,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/6a84fda3aaa04b1aa7d55630eeddcb39 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6a84fda3aaa04b1aa7d55630eeddcb39 2024-11-20T06:41:03,604 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6a84fda3aaa04b1aa7d55630eeddcb39, entries=150, sequenceid=115, filesize=11.7 K 2024-11-20T06:41:03,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/86d6e1c6f6054c639d078ce81bffbe29 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/86d6e1c6f6054c639d078ce81bffbe29 2024-11-20T06:41:03,609 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/86d6e1c6f6054c639d078ce81bffbe29, entries=150, sequenceid=115, filesize=11.7 K 2024-11-20T06:41:03,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/547cb734a78447b5b13e05c08a7116f2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/547cb734a78447b5b13e05c08a7116f2 2024-11-20T06:41:03,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084923608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084923608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,614 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/547cb734a78447b5b13e05c08a7116f2, entries=150, sequenceid=115, filesize=11.7 K 2024-11-20T06:41:03,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084923609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084923610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,616 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 792921c2784cd0f6a5ba47b7f4462f5c in 873ms, sequenceid=115, compaction requested=false 2024-11-20T06:41:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-20T06:41:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-20T06:41:03,616 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:03,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T06:41:03,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-20T06:41:03,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2510 sec 2024-11-20T06:41:03,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/803b9577184544579022d386434c6b18 is 50, key is test_row_0/A:col10/1732084863615/Put/seqid=0 2024-11-20T06:41:03,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2580 sec 2024-11-20T06:41:03,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742074_1250 (size=14541) 2024-11-20T06:41:03,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/803b9577184544579022d386434c6b18 2024-11-20T06:41:03,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/ec7da3fd3819481ca306e7a06cd8a292 is 50, key is test_row_0/B:col10/1732084863615/Put/seqid=0 2024-11-20T06:41:03,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742075_1251 (size=12151) 2024-11-20T06:41:03,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/ec7da3fd3819481ca306e7a06cd8a292 2024-11-20T06:41:03,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084923676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/f4076b58af684e04957167f79248a275 is 50, key is test_row_0/C:col10/1732084863615/Put/seqid=0 2024-11-20T06:41:03,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742076_1252 (size=12151) 2024-11-20T06:41:03,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/f4076b58af684e04957167f79248a275 2024-11-20T06:41:03,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/803b9577184544579022d386434c6b18 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/803b9577184544579022d386434c6b18 2024-11-20T06:41:03,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/803b9577184544579022d386434c6b18, entries=200, sequenceid=137, filesize=14.2 K 2024-11-20T06:41:03,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/ec7da3fd3819481ca306e7a06cd8a292 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ec7da3fd3819481ca306e7a06cd8a292 2024-11-20T06:41:03,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ec7da3fd3819481ca306e7a06cd8a292, entries=150, sequenceid=137, filesize=11.9 K 2024-11-20T06:41:03,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/f4076b58af684e04957167f79248a275 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f4076b58af684e04957167f79248a275 2024-11-20T06:41:03,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f4076b58af684e04957167f79248a275, entries=150, sequenceid=137, filesize=11.9 K 2024-11-20T06:41:03,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 792921c2784cd0f6a5ba47b7f4462f5c in 101ms, sequenceid=137, compaction requested=true 2024-11-20T06:41:03,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:03,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:03,717 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:03,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:03,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:03,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:03,717 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:03,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:03,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:03,718 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:03,718 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:03,718 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:03,718 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:03,718 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:03,718 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:03,719 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/39de4f93bc56433389a3e2fd67d6542b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6a84fda3aaa04b1aa7d55630eeddcb39, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/803b9577184544579022d386434c6b18] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=37.8 K 2024-11-20T06:41:03,719 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d9e47c61b8b14819a4838b03e96ed823, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/86d6e1c6f6054c639d078ce81bffbe29, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ec7da3fd3819481ca306e7a06cd8a292] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=35.5 K 2024-11-20T06:41:03,719 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d9e47c61b8b14819a4838b03e96ed823, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732084858943 2024-11-20T06:41:03,719 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39de4f93bc56433389a3e2fd67d6542b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732084858943 2024-11-20T06:41:03,719 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 86d6e1c6f6054c639d078ce81bffbe29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732084861440 2024-11-20T06:41:03,720 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a84fda3aaa04b1aa7d55630eeddcb39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732084861440 2024-11-20T06:41:03,720 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ec7da3fd3819481ca306e7a06cd8a292, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084863607 2024-11-20T06:41:03,720 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 803b9577184544579022d386434c6b18, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084863607 2024-11-20T06:41:03,727 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:03,727 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/9c553eb782d84096af9be98153a8d51a is 50, key is test_row_0/A:col10/1732084863615/Put/seqid=0 2024-11-20T06:41:03,728 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:03,729 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:03,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T06:41:03,729 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/0c67e0539eaf48858c5c279ec02b6547 is 50, key is test_row_0/B:col10/1732084863615/Put/seqid=0 2024-11-20T06:41:03,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/8b16e62437a941f39256cf24c326298a is 50, key is test_row_0/A:col10/1732084863727/Put/seqid=0 2024-11-20T06:41:03,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742077_1253 (size=12459) 2024-11-20T06:41:03,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084923754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084923754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084923756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084923757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,762 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/0c67e0539eaf48858c5c279ec02b6547 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0c67e0539eaf48858c5c279ec02b6547 2024-11-20T06:41:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,767 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into 0c67e0539eaf48858c5c279ec02b6547(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:03,767 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:03,767 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084863717; duration=0sec 2024-11-20T06:41:03,767 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:03,767 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:03,768 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:03,769 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:03,769 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:03,769 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:03,769 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e476f1c5f2ad48039c2f531275ee1149, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/547cb734a78447b5b13e05c08a7116f2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f4076b58af684e04957167f79248a275] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=35.5 K 2024-11-20T06:41:03,769 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting e476f1c5f2ad48039c2f531275ee1149, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732084858943 2024-11-20T06:41:03,770 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 547cb734a78447b5b13e05c08a7116f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732084861440 2024-11-20T06:41:03,770 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting f4076b58af684e04957167f79248a275, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084863607 2024-11-20T06:41:03,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742078_1254 (size=12459) 2024-11-20T06:41:03,781 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:03,781 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/75f1f417f6d1460995848fcfcfabef51 is 50, key is test_row_0/C:col10/1732084863615/Put/seqid=0 2024-11-20T06:41:03,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084923784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,789 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/9c553eb782d84096af9be98153a8d51a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/9c553eb782d84096af9be98153a8d51a 2024-11-20T06:41:03,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742079_1255 (size=12151) 2024-11-20T06:41:03,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/8b16e62437a941f39256cf24c326298a 2024-11-20T06:41:03,797 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into 9c553eb782d84096af9be98153a8d51a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:03,797 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:03,797 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084863717; duration=0sec 2024-11-20T06:41:03,797 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:03,797 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:03,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742080_1256 (size=12459) 2024-11-20T06:41:03,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/69803b9c0f404b52aef10d26f4f53975 is 50, key is test_row_0/B:col10/1732084863727/Put/seqid=0 2024-11-20T06:41:03,823 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/75f1f417f6d1460995848fcfcfabef51 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/75f1f417f6d1460995848fcfcfabef51 2024-11-20T06:41:03,829 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into 75f1f417f6d1460995848fcfcfabef51(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:03,829 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:03,829 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084863717; duration=0sec 2024-11-20T06:41:03,829 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:03,829 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:03,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742081_1257 (size=12151) 2024-11-20T06:41:03,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/69803b9c0f404b52aef10d26f4f53975 2024-11-20T06:41:03,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084923863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/f153acdcf8d946c7a7c5a0717d88ebf0 is 50, key is test_row_0/C:col10/1732084863727/Put/seqid=0 2024-11-20T06:41:03,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084923864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084923864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084923864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:03,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742082_1258 (size=12151) 2024-11-20T06:41:03,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/f153acdcf8d946c7a7c5a0717d88ebf0 2024-11-20T06:41:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/8b16e62437a941f39256cf24c326298a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/8b16e62437a941f39256cf24c326298a 2024-11-20T06:41:03,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/8b16e62437a941f39256cf24c326298a, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T06:41:03,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/69803b9c0f404b52aef10d26f4f53975 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/69803b9c0f404b52aef10d26f4f53975 2024-11-20T06:41:03,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/69803b9c0f404b52aef10d26f4f53975, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T06:41:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/f153acdcf8d946c7a7c5a0717d88ebf0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f153acdcf8d946c7a7c5a0717d88ebf0 2024-11-20T06:41:03,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f153acdcf8d946c7a7c5a0717d88ebf0, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T06:41:03,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 792921c2784cd0f6a5ba47b7f4462f5c in 171ms, sequenceid=154, compaction requested=false 2024-11-20T06:41:03,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:03,996 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:03,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:41:04,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/972ce1f63885453a900e8079e89b3348 is 50, key is test_row_0/A:col10/1732084863754/Put/seqid=0 2024-11-20T06:41:04,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742083_1259 (size=12151) 2024-11-20T06:41:04,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/972ce1f63885453a900e8079e89b3348 2024-11-20T06:41:04,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/b5f665da9bd34b2cb48e0ce1b7d85fb0 is 50, key is test_row_0/B:col10/1732084863754/Put/seqid=0 2024-11-20T06:41:04,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742084_1260 (size=12151) 2024-11-20T06:41:04,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/b5f665da9bd34b2cb48e0ce1b7d85fb0 2024-11-20T06:41:04,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084924051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/4c11955deaf042adbe702c9e5234a2f4 is 50, key is test_row_0/C:col10/1732084863754/Put/seqid=0 2024-11-20T06:41:04,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742085_1261 (size=12151) 2024-11-20T06:41:04,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084924074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084924074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084924074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084924074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084924164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,221 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T06:41:04,221 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T06:41:04,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084924373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084924384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084924384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084924384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084924384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/4c11955deaf042adbe702c9e5234a2f4 2024-11-20T06:41:04,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/972ce1f63885453a900e8079e89b3348 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/972ce1f63885453a900e8079e89b3348 2024-11-20T06:41:04,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/972ce1f63885453a900e8079e89b3348, entries=150, sequenceid=177, filesize=11.9 K 2024-11-20T06:41:04,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/b5f665da9bd34b2cb48e0ce1b7d85fb0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b5f665da9bd34b2cb48e0ce1b7d85fb0 2024-11-20T06:41:04,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b5f665da9bd34b2cb48e0ce1b7d85fb0, entries=150, sequenceid=177, filesize=11.9 K 2024-11-20T06:41:04,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/4c11955deaf042adbe702c9e5234a2f4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/4c11955deaf042adbe702c9e5234a2f4 2024-11-20T06:41:04,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/4c11955deaf042adbe702c9e5234a2f4, entries=150, sequenceid=177, filesize=11.9 K 2024-11-20T06:41:04,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 792921c2784cd0f6a5ba47b7f4462f5c in 519ms, sequenceid=177, compaction requested=true 2024-11-20T06:41:04,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:04,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:04,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:04,515 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:04,515 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:04,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:04,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:04,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:04,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:04,518 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:04,518 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:04,518 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:04,518 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:04,518 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:04,518 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:04,518 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0c67e0539eaf48858c5c279ec02b6547, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/69803b9c0f404b52aef10d26f4f53975, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b5f665da9bd34b2cb48e0ce1b7d85fb0] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=35.9 K 2024-11-20T06:41:04,518 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/9c553eb782d84096af9be98153a8d51a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/8b16e62437a941f39256cf24c326298a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/972ce1f63885453a900e8079e89b3348] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=35.9 K 2024-11-20T06:41:04,518 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c67e0539eaf48858c5c279ec02b6547, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084863607 2024-11-20T06:41:04,518 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c553eb782d84096af9be98153a8d51a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084863607 2024-11-20T06:41:04,519 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 69803b9c0f404b52aef10d26f4f53975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084863658 2024-11-20T06:41:04,519 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b16e62437a941f39256cf24c326298a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084863658 2024-11-20T06:41:04,519 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b5f665da9bd34b2cb48e0ce1b7d85fb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732084863754 2024-11-20T06:41:04,519 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 972ce1f63885453a900e8079e89b3348, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732084863754 2024-11-20T06:41:04,526 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:04,527 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:04,527 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/624f2bd573bc44b19e5302c07030b4a2 is 50, key is test_row_0/A:col10/1732084863754/Put/seqid=0 2024-11-20T06:41:04,527 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/2a6e9246dad24a998f03a2e6936eff9f is 50, key is test_row_0/B:col10/1732084863754/Put/seqid=0 2024-11-20T06:41:04,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742086_1262 (size=12561) 2024-11-20T06:41:04,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742087_1263 (size=12561) 2024-11-20T06:41:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:04,687 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:04,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T06:41:04,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/807778a9b31644dbb5d8706963bd8aa2 is 50, key is test_row_0/A:col10/1732084864685/Put/seqid=0 2024-11-20T06:41:04,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742088_1264 (size=12151) 2024-11-20T06:41:04,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/807778a9b31644dbb5d8706963bd8aa2 2024-11-20T06:41:04,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/925e74b206bb46fb8f4034f8616356a8 is 50, key is test_row_0/B:col10/1732084864685/Put/seqid=0 2024-11-20T06:41:04,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742089_1265 (size=12151) 2024-11-20T06:41:04,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084924778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084924884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084924894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084924894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084924894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084924894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:04,959 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/624f2bd573bc44b19e5302c07030b4a2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/624f2bd573bc44b19e5302c07030b4a2 2024-11-20T06:41:04,964 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/2a6e9246dad24a998f03a2e6936eff9f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/2a6e9246dad24a998f03a2e6936eff9f 2024-11-20T06:41:04,971 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into 624f2bd573bc44b19e5302c07030b4a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:04,971 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:04,972 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084864515; duration=0sec 2024-11-20T06:41:04,972 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:04,972 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:04,972 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:04,975 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:04,975 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:04,975 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:04,975 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/75f1f417f6d1460995848fcfcfabef51, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f153acdcf8d946c7a7c5a0717d88ebf0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/4c11955deaf042adbe702c9e5234a2f4] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=35.9 K 2024-11-20T06:41:04,976 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75f1f417f6d1460995848fcfcfabef51, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084863607 2024-11-20T06:41:04,976 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting f153acdcf8d946c7a7c5a0717d88ebf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084863658 2024-11-20T06:41:04,977 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c11955deaf042adbe702c9e5234a2f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732084863754 2024-11-20T06:41:04,977 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into 2a6e9246dad24a998f03a2e6936eff9f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:04,977 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:04,977 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084864515; duration=0sec 2024-11-20T06:41:04,977 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:04,977 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:04,985 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:04,985 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/9b916c011b9d4a51b54e52d6bc4e938b is 50, key is test_row_0/C:col10/1732084863754/Put/seqid=0 2024-11-20T06:41:04,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742090_1266 (size=12561) 2024-11-20T06:41:05,001 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/9b916c011b9d4a51b54e52d6bc4e938b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9b916c011b9d4a51b54e52d6bc4e938b 2024-11-20T06:41:05,010 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into 9b916c011b9d4a51b54e52d6bc4e938b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:05,010 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:05,010 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084864515; duration=0sec 2024-11-20T06:41:05,010 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:05,010 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:05,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084925094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:05,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:05,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/925e74b206bb46fb8f4034f8616356a8 2024-11-20T06:41:05,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/afbb779a8ebf4276ab86bfd13f425bdb is 50, key is test_row_0/C:col10/1732084864685/Put/seqid=0 2024-11-20T06:41:05,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742091_1267 (size=12151) 2024-11-20T06:41:05,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/afbb779a8ebf4276ab86bfd13f425bdb 2024-11-20T06:41:05,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/807778a9b31644dbb5d8706963bd8aa2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/807778a9b31644dbb5d8706963bd8aa2 2024-11-20T06:41:05,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/807778a9b31644dbb5d8706963bd8aa2, entries=150, sequenceid=194, filesize=11.9 K 2024-11-20T06:41:05,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/925e74b206bb46fb8f4034f8616356a8 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/925e74b206bb46fb8f4034f8616356a8 2024-11-20T06:41:05,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/925e74b206bb46fb8f4034f8616356a8, entries=150, sequenceid=194, filesize=11.9 K 2024-11-20T06:41:05,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/afbb779a8ebf4276ab86bfd13f425bdb as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/afbb779a8ebf4276ab86bfd13f425bdb 2024-11-20T06:41:05,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/afbb779a8ebf4276ab86bfd13f425bdb, entries=150, sequenceid=194, filesize=11.9 K 2024-11-20T06:41:05,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 792921c2784cd0f6a5ba47b7f4462f5c in 524ms, sequenceid=194, compaction requested=false 2024-11-20T06:41:05,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:05,407 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:05,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:41:05,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/3e479e96785f4f92b870431ed304990e is 50, key is test_row_0/A:col10/1732084864733/Put/seqid=0 2024-11-20T06:41:05,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742092_1268 (size=14541) 2024-11-20T06:41:05,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/3e479e96785f4f92b870431ed304990e 2024-11-20T06:41:05,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/0ad891d255224a3d9bbfde9c8e762a80 is 50, key is test_row_0/B:col10/1732084864733/Put/seqid=0 2024-11-20T06:41:05,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742093_1269 (size=12151) 2024-11-20T06:41:05,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/0ad891d255224a3d9bbfde9c8e762a80 2024-11-20T06:41:05,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/443f03cb7bba467980eb113382953dfc is 50, key is test_row_0/C:col10/1732084864733/Put/seqid=0 2024-11-20T06:41:05,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742094_1270 (size=12151) 2024-11-20T06:41:05,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/443f03cb7bba467980eb113382953dfc 2024-11-20T06:41:05,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/3e479e96785f4f92b870431ed304990e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3e479e96785f4f92b870431ed304990e 2024-11-20T06:41:05,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3e479e96785f4f92b870431ed304990e, entries=200, sequenceid=217, filesize=14.2 K 2024-11-20T06:41:05,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/0ad891d255224a3d9bbfde9c8e762a80 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0ad891d255224a3d9bbfde9c8e762a80 2024-11-20T06:41:05,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0ad891d255224a3d9bbfde9c8e762a80, entries=150, sequenceid=217, filesize=11.9 K 2024-11-20T06:41:05,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/443f03cb7bba467980eb113382953dfc as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/443f03cb7bba467980eb113382953dfc 2024-11-20T06:41:05,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084925503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:05,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/443f03cb7bba467980eb113382953dfc, entries=150, sequenceid=217, filesize=11.9 K 2024-11-20T06:41:05,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 792921c2784cd0f6a5ba47b7f4462f5c in 114ms, sequenceid=217, compaction requested=true 2024-11-20T06:41:05,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:05,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:05,522 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:05,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:05,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:05,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:05,522 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:05,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:05,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:05,523 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:05,523 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:05,523 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:05,523 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/2a6e9246dad24a998f03a2e6936eff9f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/925e74b206bb46fb8f4034f8616356a8, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0ad891d255224a3d9bbfde9c8e762a80] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.0 K 2024-11-20T06:41:05,523 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:05,523 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:05,523 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:05,524 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/624f2bd573bc44b19e5302c07030b4a2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/807778a9b31644dbb5d8706963bd8aa2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3e479e96785f4f92b870431ed304990e] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=38.3 K 2024-11-20T06:41:05,524 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a6e9246dad24a998f03a2e6936eff9f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732084863754 2024-11-20T06:41:05,525 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624f2bd573bc44b19e5302c07030b4a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732084863754 2024-11-20T06:41:05,525 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 925e74b206bb46fb8f4034f8616356a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732084864044 2024-11-20T06:41:05,525 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 807778a9b31644dbb5d8706963bd8aa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732084864044 2024-11-20T06:41:05,526 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ad891d255224a3d9bbfde9c8e762a80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732084864733 2024-11-20T06:41:05,526 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e479e96785f4f92b870431ed304990e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732084864733 2024-11-20T06:41:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T06:41:05,534 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:05,535 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-20T06:41:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T06:41:05,538 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:05,540 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:05,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:05,546 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:05,547 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/3f7940d1536a49649e48ea87870db754 is 50, key is test_row_0/B:col10/1732084864733/Put/seqid=0 2024-11-20T06:41:05,565 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:05,566 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/4aa3ce84b9e4425b91a8344c005263a3 is 50, key is test_row_0/A:col10/1732084864733/Put/seqid=0 2024-11-20T06:41:05,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742095_1271 (size=12663) 2024-11-20T06:41:05,602 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/3f7940d1536a49649e48ea87870db754 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/3f7940d1536a49649e48ea87870db754 2024-11-20T06:41:05,610 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into 3f7940d1536a49649e48ea87870db754(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:05,611 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:05,611 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084865522; duration=0sec 2024-11-20T06:41:05,611 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:05,611 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:05,611 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:05,613 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:05,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742096_1272 (size=12663) 2024-11-20T06:41:05,613 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:05,613 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:05,614 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9b916c011b9d4a51b54e52d6bc4e938b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/afbb779a8ebf4276ab86bfd13f425bdb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/443f03cb7bba467980eb113382953dfc] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.0 K 2024-11-20T06:41:05,614 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b916c011b9d4a51b54e52d6bc4e938b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732084863754 2024-11-20T06:41:05,614 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting afbb779a8ebf4276ab86bfd13f425bdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732084864044 2024-11-20T06:41:05,616 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 443f03cb7bba467980eb113382953dfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732084864733 2024-11-20T06:41:05,619 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/4aa3ce84b9e4425b91a8344c005263a3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4aa3ce84b9e4425b91a8344c005263a3 2024-11-20T06:41:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:05,627 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:05,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T06:41:05,630 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into 4aa3ce84b9e4425b91a8344c005263a3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:05,630 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:05,630 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084865521; duration=0sec 2024-11-20T06:41:05,630 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:05,630 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:05,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/0c18371a5da94cd8a9b9fe1ff719ce71 is 50, key is test_row_0/A:col10/1732084865501/Put/seqid=0 2024-11-20T06:41:05,642 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#233 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:05,643 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/66896c426b844d1c96951ebc7e09b56f is 50, key is test_row_0/C:col10/1732084864733/Put/seqid=0 2024-11-20T06:41:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T06:41:05,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742097_1273 (size=14541) 2024-11-20T06:41:05,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/0c18371a5da94cd8a9b9fe1ff719ce71 2024-11-20T06:41:05,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742098_1274 (size=12663) 2024-11-20T06:41:05,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/e6d9949df5ae41c3af369e1a5b268a5e is 50, key is test_row_0/B:col10/1732084865501/Put/seqid=0 2024-11-20T06:41:05,693 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/66896c426b844d1c96951ebc7e09b56f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/66896c426b844d1c96951ebc7e09b56f 2024-11-20T06:41:05,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T06:41:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:05,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:05,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:05,704 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into 66896c426b844d1c96951ebc7e09b56f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:05,704 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:05,704 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084865522; duration=0sec 2024-11-20T06:41:05,704 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:05,704 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:05,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742099_1275 (size=12151) 2024-11-20T06:41:05,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/e6d9949df5ae41c3af369e1a5b268a5e 2024-11-20T06:41:05,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/95ea88dca63843d09e8ee323ebca2c44 is 50, key is test_row_0/C:col10/1732084865501/Put/seqid=0 2024-11-20T06:41:05,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084925729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:05,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742100_1276 (size=12151) 2024-11-20T06:41:05,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/95ea88dca63843d09e8ee323ebca2c44 2024-11-20T06:41:05,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/0c18371a5da94cd8a9b9fe1ff719ce71 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/0c18371a5da94cd8a9b9fe1ff719ce71 2024-11-20T06:41:05,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/0c18371a5da94cd8a9b9fe1ff719ce71, entries=200, sequenceid=235, filesize=14.2 K 2024-11-20T06:41:05,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/e6d9949df5ae41c3af369e1a5b268a5e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e6d9949df5ae41c3af369e1a5b268a5e 2024-11-20T06:41:05,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e6d9949df5ae41c3af369e1a5b268a5e, entries=150, sequenceid=235, filesize=11.9 K 2024-11-20T06:41:05,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/95ea88dca63843d09e8ee323ebca2c44 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95ea88dca63843d09e8ee323ebca2c44 2024-11-20T06:41:05,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95ea88dca63843d09e8ee323ebca2c44, entries=150, sequenceid=235, filesize=11.9 K 2024-11-20T06:41:05,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 792921c2784cd0f6a5ba47b7f4462f5c in 139ms, sequenceid=235, compaction requested=false 2024-11-20T06:41:05,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:05,847 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:05,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:41:05,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T06:41:05,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:05,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:05,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:05,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:05,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T06:41:05,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c4371b6897f1436a9cb356d9dbcc9ff5 is 50, key is test_row_0/A:col10/1732084865727/Put/seqid=0 2024-11-20T06:41:05,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742101_1277 (size=14591) 2024-11-20T06:41:05,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084925904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:05,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084925905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:05,911 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 792921c2784cd0f6a5ba47b7f4462f5c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084925911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084925910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084925912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T06:41:06,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:06,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:06,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084926014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084926023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084926023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084926023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T06:41:06,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:06,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:06,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T06:41:06,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084926223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084926234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084926234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084926234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c4371b6897f1436a9cb356d9dbcc9ff5 2024-11-20T06:41:06,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/8b5d64e8e0114667b56ea4ea74d02f89 is 50, key is test_row_0/B:col10/1732084865727/Put/seqid=0 2024-11-20T06:41:06,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742102_1278 (size=12201) 2024-11-20T06:41:06,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/8b5d64e8e0114667b56ea4ea74d02f89 2024-11-20T06:41:06,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T06:41:06,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:06,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:06,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:06,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ad693bb5313540f6989861ec7d65eeb9 is 50, key is test_row_0/C:col10/1732084865727/Put/seqid=0 2024-11-20T06:41:06,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742103_1279 (size=12201) 2024-11-20T06:41:06,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ad693bb5313540f6989861ec7d65eeb9 2024-11-20T06:41:06,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c4371b6897f1436a9cb356d9dbcc9ff5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c4371b6897f1436a9cb356d9dbcc9ff5 2024-11-20T06:41:06,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c4371b6897f1436a9cb356d9dbcc9ff5, entries=200, sequenceid=257, filesize=14.2 K 2024-11-20T06:41:06,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/8b5d64e8e0114667b56ea4ea74d02f89 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8b5d64e8e0114667b56ea4ea74d02f89 2024-11-20T06:41:06,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8b5d64e8e0114667b56ea4ea74d02f89, entries=150, sequenceid=257, filesize=11.9 K 2024-11-20T06:41:06,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ad693bb5313540f6989861ec7d65eeb9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad693bb5313540f6989861ec7d65eeb9 2024-11-20T06:41:06,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad693bb5313540f6989861ec7d65eeb9, entries=150, sequenceid=257, filesize=11.9 K 2024-11-20T06:41:06,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 792921c2784cd0f6a5ba47b7f4462f5c in 535ms, sequenceid=257, compaction requested=true 2024-11-20T06:41:06,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:06,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:06,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:06,382 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:06,382 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:06,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:06,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:06,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:06,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:06,383 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:06,384 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:06,384 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,384 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4aa3ce84b9e4425b91a8344c005263a3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/0c18371a5da94cd8a9b9fe1ff719ce71, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c4371b6897f1436a9cb356d9dbcc9ff5] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=40.8 K 2024-11-20T06:41:06,384 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aa3ce84b9e4425b91a8344c005263a3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732084864733 2024-11-20T06:41:06,384 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:06,384 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:06,385 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,385 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/3f7940d1536a49649e48ea87870db754, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e6d9949df5ae41c3af369e1a5b268a5e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8b5d64e8e0114667b56ea4ea74d02f89] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.1 K 2024-11-20T06:41:06,386 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c18371a5da94cd8a9b9fe1ff719ce71, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732084865487 2024-11-20T06:41:06,386 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f7940d1536a49649e48ea87870db754, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732084864733 2024-11-20T06:41:06,386 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4371b6897f1436a9cb356d9dbcc9ff5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084865704 2024-11-20T06:41:06,386 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting e6d9949df5ae41c3af369e1a5b268a5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732084865494 2024-11-20T06:41:06,386 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b5d64e8e0114667b56ea4ea74d02f89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084865704 2024-11-20T06:41:06,396 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:06,396 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:06,396 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/659477d78152413abfd962baf398a89b is 50, key is test_row_0/A:col10/1732084865727/Put/seqid=0 2024-11-20T06:41:06,398 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/d721eef1194c44c981a5267fca41a5a5 is 50, key is test_row_0/B:col10/1732084865727/Put/seqid=0 2024-11-20T06:41:06,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742104_1280 (size=12815) 2024-11-20T06:41:06,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742105_1281 (size=12815) 2024-11-20T06:41:06,425 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/d721eef1194c44c981a5267fca41a5a5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d721eef1194c44c981a5267fca41a5a5 2024-11-20T06:41:06,431 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into d721eef1194c44c981a5267fca41a5a5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:06,431 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:06,432 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084866382; duration=0sec 2024-11-20T06:41:06,432 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:06,432 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:06,432 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:06,433 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:06,433 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:06,433 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,433 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/66896c426b844d1c96951ebc7e09b56f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95ea88dca63843d09e8ee323ebca2c44, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad693bb5313540f6989861ec7d65eeb9] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.1 K 2024-11-20T06:41:06,434 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 66896c426b844d1c96951ebc7e09b56f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732084864733 2024-11-20T06:41:06,435 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ea88dca63843d09e8ee323ebca2c44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732084865494 2024-11-20T06:41:06,435 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ad693bb5313540f6989861ec7d65eeb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084865704 2024-11-20T06:41:06,448 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:06,449 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/cc4a6e38de4e4c07a7bead47823e4a60 is 50, key is test_row_0/C:col10/1732084865727/Put/seqid=0 2024-11-20T06:41:06,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742106_1282 (size=12815) 2024-11-20T06:41:06,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T06:41:06,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,463 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T06:41:06,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c21dcd706eff47039f21d11875a840a2 is 50, key is test_row_0/A:col10/1732084865883/Put/seqid=0 2024-11-20T06:41:06,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742107_1283 (size=12301) 2024-11-20T06:41:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:06,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:06,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084926562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084926563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084926563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084926566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T06:41:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084926673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084926673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084926674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084926674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,818 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/659477d78152413abfd962baf398a89b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/659477d78152413abfd962baf398a89b 2024-11-20T06:41:06,824 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into 659477d78152413abfd962baf398a89b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:06,824 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:06,824 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084866382; duration=0sec 2024-11-20T06:41:06,824 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:06,824 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:06,861 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/cc4a6e38de4e4c07a7bead47823e4a60 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cc4a6e38de4e4c07a7bead47823e4a60 2024-11-20T06:41:06,865 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into cc4a6e38de4e4c07a7bead47823e4a60(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:06,865 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:06,865 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084866383; duration=0sec 2024-11-20T06:41:06,865 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:06,865 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:06,877 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c21dcd706eff47039f21d11875a840a2 2024-11-20T06:41:06,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/c1b1ec7654154a54b70cb69a6d1af0e4 is 50, key is test_row_0/B:col10/1732084865883/Put/seqid=0 2024-11-20T06:41:06,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084926883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084926883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084926883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084926883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:06,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:06,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:06,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742108_1284 (size=12301) 2024-11-20T06:41:06,888 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/c1b1ec7654154a54b70cb69a6d1af0e4 2024-11-20T06:41:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/c0e12c35ebbf4254a6ace621db99a217 is 50, key is test_row_0/C:col10/1732084865883/Put/seqid=0 2024-11-20T06:41:06,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742109_1285 (size=12301) 2024-11-20T06:41:06,898 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/c0e12c35ebbf4254a6ace621db99a217 2024-11-20T06:41:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/c21dcd706eff47039f21d11875a840a2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c21dcd706eff47039f21d11875a840a2 2024-11-20T06:41:06,908 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c21dcd706eff47039f21d11875a840a2, entries=150, sequenceid=275, filesize=12.0 K 2024-11-20T06:41:06,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/c1b1ec7654154a54b70cb69a6d1af0e4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/c1b1ec7654154a54b70cb69a6d1af0e4 2024-11-20T06:41:06,914 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/c1b1ec7654154a54b70cb69a6d1af0e4, entries=150, sequenceid=275, filesize=12.0 K 2024-11-20T06:41:06,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/c0e12c35ebbf4254a6ace621db99a217 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c0e12c35ebbf4254a6ace621db99a217 2024-11-20T06:41:06,922 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c0e12c35ebbf4254a6ace621db99a217, entries=150, sequenceid=275, filesize=12.0 K 2024-11-20T06:41:06,922 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 792921c2784cd0f6a5ba47b7f4462f5c in 459ms, sequenceid=275, compaction requested=false 2024-11-20T06:41:06,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:06,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:06,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-20T06:41:06,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-20T06:41:06,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-20T06:41:06,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3840 sec 2024-11-20T06:41:06,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.3910 sec 2024-11-20T06:41:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:07,197 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:07,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:41:07,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/7c84d8b6588f4cda89bc0629f40f7274 is 50, key is test_row_0/A:col10/1732084867196/Put/seqid=0 2024-11-20T06:41:07,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742110_1286 (size=12301) 2024-11-20T06:41:07,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084927208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084927208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084927209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084927212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084927323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084927323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084927324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084927324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084927533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084927533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084927533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084927534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/7c84d8b6588f4cda89bc0629f40f7274 2024-11-20T06:41:07,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/e8d6c8d3fdcd448a998236e82748edd3 is 50, key is test_row_0/B:col10/1732084867196/Put/seqid=0 2024-11-20T06:41:07,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742111_1287 (size=12301) 2024-11-20T06:41:07,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/e8d6c8d3fdcd448a998236e82748edd3 2024-11-20T06:41:07,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ad4a3f04852541879ebdfd122bb54970 is 50, key is test_row_0/C:col10/1732084867196/Put/seqid=0 2024-11-20T06:41:07,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742112_1288 (size=12301) 2024-11-20T06:41:07,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T06:41:07,684 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:07,684 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-20T06:41:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T06:41:07,686 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:07,687 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:07,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:07,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T06:41:07,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T06:41:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:07,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:07,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:07,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084927844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084927844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084927844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084927844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:07,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084927924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:07,926 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 792921c2784cd0f6a5ba47b7f4462f5c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:41:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:07,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T06:41:07,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:07,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:07,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:07,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:07,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:07,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T06:41:08,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ad4a3f04852541879ebdfd122bb54970 2024-11-20T06:41:08,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/7c84d8b6588f4cda89bc0629f40f7274 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/7c84d8b6588f4cda89bc0629f40f7274 2024-11-20T06:41:08,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/7c84d8b6588f4cda89bc0629f40f7274, entries=150, sequenceid=298, filesize=12.0 K 2024-11-20T06:41:08,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/e8d6c8d3fdcd448a998236e82748edd3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e8d6c8d3fdcd448a998236e82748edd3 2024-11-20T06:41:08,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e8d6c8d3fdcd448a998236e82748edd3, entries=150, sequenceid=298, filesize=12.0 K 2024-11-20T06:41:08,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ad4a3f04852541879ebdfd122bb54970 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad4a3f04852541879ebdfd122bb54970 2024-11-20T06:41:08,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad4a3f04852541879ebdfd122bb54970, entries=150, sequenceid=298, filesize=12.0 K 2024-11-20T06:41:08,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 792921c2784cd0f6a5ba47b7f4462f5c in 847ms, sequenceid=298, compaction requested=true 2024-11-20T06:41:08,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:08,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:08,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:08,045 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:08,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:08,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:08,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:08,045 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:08,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:08,045 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:08,045 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:08,045 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:08,046 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/659477d78152413abfd962baf398a89b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c21dcd706eff47039f21d11875a840a2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/7c84d8b6588f4cda89bc0629f40f7274] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.5 K 2024-11-20T06:41:08,046 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:08,046 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:08,046 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:08,046 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d721eef1194c44c981a5267fca41a5a5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/c1b1ec7654154a54b70cb69a6d1af0e4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e8d6c8d3fdcd448a998236e82748edd3] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.5 K 2024-11-20T06:41:08,046 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 659477d78152413abfd962baf398a89b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084865704 2024-11-20T06:41:08,046 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d721eef1194c44c981a5267fca41a5a5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084865704 2024-11-20T06:41:08,046 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting c21dcd706eff47039f21d11875a840a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732084865883 2024-11-20T06:41:08,046 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c1b1ec7654154a54b70cb69a6d1af0e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732084865883 2024-11-20T06:41:08,046 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c84d8b6588f4cda89bc0629f40f7274, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732084866562 2024-11-20T06:41:08,047 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d6c8d3fdcd448a998236e82748edd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732084866562 2024-11-20T06:41:08,053 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:08,053 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:08,053 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/ad0a55344ae84f3ba7147317813ec7ee is 50, key is test_row_0/B:col10/1732084867196/Put/seqid=0 2024-11-20T06:41:08,053 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/9b5f7575974d49afabac0850b28ae7f0 is 50, key is test_row_0/A:col10/1732084867196/Put/seqid=0 2024-11-20T06:41:08,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742113_1289 (size=13017) 2024-11-20T06:41:08,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742114_1290 (size=13017) 2024-11-20T06:41:08,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T06:41:08,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:08,145 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T06:41:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/d53253f126e54d40993690f0ea985097 is 50, key is test_row_0/A:col10/1732084867208/Put/seqid=0 2024-11-20T06:41:08,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742115_1291 (size=12301) 2024-11-20T06:41:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T06:41:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:08,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084928371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084928372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084928373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084928376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,469 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/ad0a55344ae84f3ba7147317813ec7ee as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ad0a55344ae84f3ba7147317813ec7ee 2024-11-20T06:41:08,471 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/9b5f7575974d49afabac0850b28ae7f0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/9b5f7575974d49afabac0850b28ae7f0 2024-11-20T06:41:08,473 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into ad0a55344ae84f3ba7147317813ec7ee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:08,473 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:08,473 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084868045; duration=0sec 2024-11-20T06:41:08,473 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:08,473 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:08,473 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:08,474 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:08,474 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:08,474 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:08,474 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cc4a6e38de4e4c07a7bead47823e4a60, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c0e12c35ebbf4254a6ace621db99a217, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad4a3f04852541879ebdfd122bb54970] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.5 K 2024-11-20T06:41:08,474 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting cc4a6e38de4e4c07a7bead47823e4a60, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084865704 2024-11-20T06:41:08,474 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c0e12c35ebbf4254a6ace621db99a217, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732084865883 2024-11-20T06:41:08,475 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ad4a3f04852541879ebdfd122bb54970, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732084866562 2024-11-20T06:41:08,475 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into 9b5f7575974d49afabac0850b28ae7f0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:08,475 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:08,475 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084868044; duration=0sec 2024-11-20T06:41:08,476 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:08,476 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:08,480 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:08,481 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/f0df072cbdaf494ba53929fdf0fcb767 is 50, key is test_row_0/C:col10/1732084867196/Put/seqid=0 2024-11-20T06:41:08,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742116_1292 (size=13017) 2024-11-20T06:41:08,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084928484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084928484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084928484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084928484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,489 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/f0df072cbdaf494ba53929fdf0fcb767 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f0df072cbdaf494ba53929fdf0fcb767 2024-11-20T06:41:08,494 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into f0df072cbdaf494ba53929fdf0fcb767(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:08,494 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:08,494 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084868045; duration=0sec 2024-11-20T06:41:08,494 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:08,494 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:08,556 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/d53253f126e54d40993690f0ea985097 2024-11-20T06:41:08,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/b6fb55f039a74a6fa4bb161a8a6072a2 is 50, key is test_row_0/B:col10/1732084867208/Put/seqid=0 2024-11-20T06:41:08,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742117_1293 (size=12301) 2024-11-20T06:41:08,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084928694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084928694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084928694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084928694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T06:41:08,967 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/b6fb55f039a74a6fa4bb161a8a6072a2 2024-11-20T06:41:08,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/8f0573648e1340afbd1a8ef1781be35f is 50, key is test_row_0/C:col10/1732084867208/Put/seqid=0 2024-11-20T06:41:08,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742118_1294 (size=12301) 2024-11-20T06:41:09,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084929004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084929004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084929004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084929004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,378 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/8f0573648e1340afbd1a8ef1781be35f 2024-11-20T06:41:09,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/d53253f126e54d40993690f0ea985097 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/d53253f126e54d40993690f0ea985097 2024-11-20T06:41:09,387 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/d53253f126e54d40993690f0ea985097, entries=150, sequenceid=313, filesize=12.0 K 2024-11-20T06:41:09,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/b6fb55f039a74a6fa4bb161a8a6072a2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b6fb55f039a74a6fa4bb161a8a6072a2 2024-11-20T06:41:09,391 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b6fb55f039a74a6fa4bb161a8a6072a2, entries=150, sequenceid=313, filesize=12.0 K 2024-11-20T06:41:09,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/8f0573648e1340afbd1a8ef1781be35f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8f0573648e1340afbd1a8ef1781be35f 2024-11-20T06:41:09,395 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8f0573648e1340afbd1a8ef1781be35f, entries=150, sequenceid=313, filesize=12.0 K 2024-11-20T06:41:09,421 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 792921c2784cd0f6a5ba47b7f4462f5c in 1276ms, sequenceid=313, compaction requested=false 2024-11-20T06:41:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:09,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-20T06:41:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-20T06:41:09,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-20T06:41:09,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7360 sec 2024-11-20T06:41:09,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.7410 sec 2024-11-20T06:41:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:09,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T06:41:09,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/b6ae5561942742308e25b1ba302a4e8a is 50, key is test_row_0/A:col10/1732084869516/Put/seqid=0 2024-11-20T06:41:09,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742119_1295 (size=12301) 2024-11-20T06:41:09,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084929525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084929527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084929527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084929528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084929634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084929634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084929634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084929634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T06:41:09,834 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:09,834 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-20T06:41:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T06:41:09,836 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:09,836 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:09,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:09,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084929844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084929844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084929844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084929844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:09,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/b6ae5561942742308e25b1ba302a4e8a 2024-11-20T06:41:09,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/dacdbe47a3554b32b0a985f50e031b1b is 50, key is test_row_0/B:col10/1732084869516/Put/seqid=0 2024-11-20T06:41:09,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742120_1296 (size=12301) 2024-11-20T06:41:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T06:41:09,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T06:41:09,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:09,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:09,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:09,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:09,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:10,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T06:41:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:10,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T06:41:10,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084930154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084930154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084930154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084930154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T06:41:10,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:10,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:10,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:10,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:10,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:10,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/dacdbe47a3554b32b0a985f50e031b1b 2024-11-20T06:41:10,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/69b6ce941c5b425387d769841c340336 is 50, key is test_row_0/C:col10/1732084869516/Put/seqid=0 2024-11-20T06:41:10,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742121_1297 (size=12301) 2024-11-20T06:41:10,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/69b6ce941c5b425387d769841c340336 2024-11-20T06:41:10,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/b6ae5561942742308e25b1ba302a4e8a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/b6ae5561942742308e25b1ba302a4e8a 2024-11-20T06:41:10,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/b6ae5561942742308e25b1ba302a4e8a, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T06:41:10,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/dacdbe47a3554b32b0a985f50e031b1b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/dacdbe47a3554b32b0a985f50e031b1b 2024-11-20T06:41:10,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/dacdbe47a3554b32b0a985f50e031b1b, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T06:41:10,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/69b6ce941c5b425387d769841c340336 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/69b6ce941c5b425387d769841c340336 2024-11-20T06:41:10,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/69b6ce941c5b425387d769841c340336, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T06:41:10,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 792921c2784cd0f6a5ba47b7f4462f5c in 848ms, sequenceid=339, compaction requested=true 2024-11-20T06:41:10,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:10,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:10,365 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:10,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:10,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:10,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:10,365 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:10,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:10,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:10,366 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:10,366 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:10,366 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:10,366 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/9b5f7575974d49afabac0850b28ae7f0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/d53253f126e54d40993690f0ea985097, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/b6ae5561942742308e25b1ba302a4e8a] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.7 K 2024-11-20T06:41:10,366 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:10,366 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:10,366 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:10,366 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ad0a55344ae84f3ba7147317813ec7ee, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b6fb55f039a74a6fa4bb161a8a6072a2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/dacdbe47a3554b32b0a985f50e031b1b] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.7 K 2024-11-20T06:41:10,366 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b5f7575974d49afabac0850b28ae7f0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732084866562 2024-11-20T06:41:10,366 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ad0a55344ae84f3ba7147317813ec7ee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732084866562 2024-11-20T06:41:10,366 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting d53253f126e54d40993690f0ea985097, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732084867207 2024-11-20T06:41:10,366 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b6fb55f039a74a6fa4bb161a8a6072a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732084867207 2024-11-20T06:41:10,367 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6ae5561942742308e25b1ba302a4e8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084868375 2024-11-20T06:41:10,367 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting dacdbe47a3554b32b0a985f50e031b1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084868375 2024-11-20T06:41:10,373 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:10,374 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/06bf45f724b84eb48c5c162e8661369e is 50, key is test_row_0/A:col10/1732084869516/Put/seqid=0 2024-11-20T06:41:10,374 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:10,374 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/9f7c197b354348b4a31c72c92492dc31 is 50, key is test_row_0/B:col10/1732084869516/Put/seqid=0 2024-11-20T06:41:10,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742122_1298 (size=13119) 2024-11-20T06:41:10,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742123_1299 (size=13119) 2024-11-20T06:41:10,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T06:41:10,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:10,446 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T06:41:10,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/3b8b2ccba9c5409eb58e0d72d99051ab is 50, key is test_row_0/A:col10/1732084869520/Put/seqid=0 2024-11-20T06:41:10,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742124_1300 (size=12301) 2024-11-20T06:41:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T06:41:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:10,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:10,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084930696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084930696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084930697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084930698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,795 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/9f7c197b354348b4a31c72c92492dc31 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/9f7c197b354348b4a31c72c92492dc31 2024-11-20T06:41:10,800 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into 9f7c197b354348b4a31c72c92492dc31(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:10,800 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:10,800 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084870365; duration=0sec 2024-11-20T06:41:10,801 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:10,801 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:10,801 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:10,803 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/06bf45f724b84eb48c5c162e8661369e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/06bf45f724b84eb48c5c162e8661369e 2024-11-20T06:41:10,803 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:10,803 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:10,803 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:10,803 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f0df072cbdaf494ba53929fdf0fcb767, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8f0573648e1340afbd1a8ef1781be35f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/69b6ce941c5b425387d769841c340336] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.7 K 2024-11-20T06:41:10,803 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting f0df072cbdaf494ba53929fdf0fcb767, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732084866562 2024-11-20T06:41:10,804 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f0573648e1340afbd1a8ef1781be35f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732084867207 2024-11-20T06:41:10,804 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b6ce941c5b425387d769841c340336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084868375 2024-11-20T06:41:10,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084930804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084930804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,813 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into 06bf45f724b84eb48c5c162e8661369e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:10,813 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:10,813 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084870365; duration=0sec 2024-11-20T06:41:10,813 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:10,813 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:10,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084930814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084930814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:10,826 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:10,826 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/e2a095963dd144a2a1ff9f0ae221d53c is 50, key is test_row_0/C:col10/1732084869516/Put/seqid=0 2024-11-20T06:41:10,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742125_1301 (size=13119) 2024-11-20T06:41:10,856 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/3b8b2ccba9c5409eb58e0d72d99051ab 2024-11-20T06:41:10,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/d0cd5980f0e44dc6b2b6614632853e3a is 50, key is test_row_0/B:col10/1732084869520/Put/seqid=0 2024-11-20T06:41:10,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742126_1302 (size=12301) 2024-11-20T06:41:10,900 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/d0cd5980f0e44dc6b2b6614632853e3a 2024-11-20T06:41:10,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/e482706c11ab479880efbd02db43b979 is 50, key is test_row_0/C:col10/1732084869520/Put/seqid=0 2024-11-20T06:41:10,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742127_1303 (size=12301) 2024-11-20T06:41:10,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T06:41:11,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084931014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084931014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084931023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084931023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,250 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/e2a095963dd144a2a1ff9f0ae221d53c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e2a095963dd144a2a1ff9f0ae221d53c 2024-11-20T06:41:11,259 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into e2a095963dd144a2a1ff9f0ae221d53c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:11,259 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:11,259 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084870365; duration=0sec 2024-11-20T06:41:11,259 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:11,259 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:11,325 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/e482706c11ab479880efbd02db43b979 2024-11-20T06:41:11,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084931324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084931324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084931334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084931334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/3b8b2ccba9c5409eb58e0d72d99051ab as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3b8b2ccba9c5409eb58e0d72d99051ab 2024-11-20T06:41:11,343 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3b8b2ccba9c5409eb58e0d72d99051ab, entries=150, sequenceid=352, filesize=12.0 K 2024-11-20T06:41:11,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/d0cd5980f0e44dc6b2b6614632853e3a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d0cd5980f0e44dc6b2b6614632853e3a 2024-11-20T06:41:11,351 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d0cd5980f0e44dc6b2b6614632853e3a, entries=150, sequenceid=352, filesize=12.0 K 2024-11-20T06:41:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/e482706c11ab479880efbd02db43b979 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e482706c11ab479880efbd02db43b979 2024-11-20T06:41:11,371 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e482706c11ab479880efbd02db43b979, entries=150, sequenceid=352, filesize=12.0 K 2024-11-20T06:41:11,373 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 792921c2784cd0f6a5ba47b7f4462f5c in 927ms, sequenceid=352, compaction requested=false 2024-11-20T06:41:11,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:11,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:11,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-20T06:41:11,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-20T06:41:11,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-20T06:41:11,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5390 sec 2024-11-20T06:41:11,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5440 sec 2024-11-20T06:41:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:11,836 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:11,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T06:41:11,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/33227c761a3148ffbe434ccd737ebc15 is 50, key is test_row_0/A:col10/1732084871835/Put/seqid=0 2024-11-20T06:41:11,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742128_1304 (size=14741) 2024-11-20T06:41:11,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/33227c761a3148ffbe434ccd737ebc15 2024-11-20T06:41:11,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084931847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084931847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084931848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084931850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/e84a63e80eac4924a3a02696a3ce7f4a is 50, key is test_row_0/B:col10/1732084871835/Put/seqid=0 2024-11-20T06:41:11,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742129_1305 (size=12301) 2024-11-20T06:41:11,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45176 deadline: 1732084931953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084931954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084931954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084931954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,956 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 792921c2784cd0f6a5ba47b7f4462f5c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:41:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084931964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:11,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T06:41:11,984 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:11,984 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:11,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-20T06:41:11,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T06:41:11,986 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:11,987 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:11,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T06:41:12,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T06:41:12,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:12,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:12,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084932164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084932164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:12,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084932164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:12,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084932174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:12,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:12,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/e84a63e80eac4924a3a02696a3ce7f4a 2024-11-20T06:41:12,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/6b103e83f3964fbea64286189147f14b is 50, key is test_row_0/C:col10/1732084871835/Put/seqid=0 2024-11-20T06:41:12,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742130_1306 (size=12301) 2024-11-20T06:41:12,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T06:41:12,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:12,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T06:41:12,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T06:41:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084932473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084932473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084932474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:12,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084932484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:12,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T06:41:12,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:12,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T06:41:12,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/6b103e83f3964fbea64286189147f14b 2024-11-20T06:41:12,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/33227c761a3148ffbe434ccd737ebc15 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/33227c761a3148ffbe434ccd737ebc15 2024-11-20T06:41:12,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/33227c761a3148ffbe434ccd737ebc15, entries=200, sequenceid=380, filesize=14.4 K 2024-11-20T06:41:12,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/e84a63e80eac4924a3a02696a3ce7f4a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e84a63e80eac4924a3a02696a3ce7f4a 2024-11-20T06:41:12,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e84a63e80eac4924a3a02696a3ce7f4a, entries=150, sequenceid=380, filesize=12.0 K 2024-11-20T06:41:12,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/6b103e83f3964fbea64286189147f14b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/6b103e83f3964fbea64286189147f14b 2024-11-20T06:41:12,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/6b103e83f3964fbea64286189147f14b, entries=150, sequenceid=380, filesize=12.0 K 2024-11-20T06:41:12,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 792921c2784cd0f6a5ba47b7f4462f5c in 850ms, sequenceid=380, compaction requested=true 2024-11-20T06:41:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:12,686 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:12,686 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:12,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:12,687 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:12,687 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:12,687 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:12,687 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:12,687 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,687 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,687 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/06bf45f724b84eb48c5c162e8661369e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3b8b2ccba9c5409eb58e0d72d99051ab, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/33227c761a3148ffbe434ccd737ebc15] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=39.2 K 2024-11-20T06:41:12,687 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/9f7c197b354348b4a31c72c92492dc31, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d0cd5980f0e44dc6b2b6614632853e3a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e84a63e80eac4924a3a02696a3ce7f4a] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.8 K 2024-11-20T06:41:12,687 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f7c197b354348b4a31c72c92492dc31, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084868375 2024-11-20T06:41:12,687 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06bf45f724b84eb48c5c162e8661369e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084868375 2024-11-20T06:41:12,687 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b8b2ccba9c5409eb58e0d72d99051ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732084869520 2024-11-20T06:41:12,688 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d0cd5980f0e44dc6b2b6614632853e3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732084869520 2024-11-20T06:41:12,688 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33227c761a3148ffbe434ccd737ebc15, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732084870695 2024-11-20T06:41:12,688 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting e84a63e80eac4924a3a02696a3ce7f4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732084870697 2024-11-20T06:41:12,693 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:12,694 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/5b567b9df3724a92905326cbcf0f5df0 is 50, key is test_row_0/B:col10/1732084871835/Put/seqid=0 2024-11-20T06:41:12,696 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:12,696 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/46a69f6e9cd246cc95a18dd978837fce is 50, key is test_row_0/A:col10/1732084871835/Put/seqid=0 2024-11-20T06:41:12,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742131_1307 (size=13221) 2024-11-20T06:41:12,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742132_1308 (size=13221) 2024-11-20T06:41:12,703 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/46a69f6e9cd246cc95a18dd978837fce as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/46a69f6e9cd246cc95a18dd978837fce 2024-11-20T06:41:12,708 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into 46a69f6e9cd246cc95a18dd978837fce(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:12,708 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:12,708 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084872686; duration=0sec 2024-11-20T06:41:12,708 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:12,708 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:12,708 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:12,709 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:12,709 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:12,709 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,710 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e2a095963dd144a2a1ff9f0ae221d53c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e482706c11ab479880efbd02db43b979, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/6b103e83f3964fbea64286189147f14b] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.8 K 2024-11-20T06:41:12,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2a095963dd144a2a1ff9f0ae221d53c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732084868375 2024-11-20T06:41:12,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting e482706c11ab479880efbd02db43b979, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732084869520 2024-11-20T06:41:12,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b103e83f3964fbea64286189147f14b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732084870697 2024-11-20T06:41:12,717 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:12,718 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ca0ae88bfa3644dcb18a17ab3cabdc61 is 50, key is test_row_0/C:col10/1732084871835/Put/seqid=0 2024-11-20T06:41:12,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742133_1309 (size=13221) 2024-11-20T06:41:12,742 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ca0ae88bfa3644dcb18a17ab3cabdc61 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ca0ae88bfa3644dcb18a17ab3cabdc61 2024-11-20T06:41:12,746 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into ca0ae88bfa3644dcb18a17ab3cabdc61(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:12,746 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:12,746 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084872686; duration=0sec 2024-11-20T06:41:12,746 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:12,746 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:12,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T06:41:12,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,749 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:41:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/f612c67d31d643e199c834f2c49a2690 is 50, key is test_row_0/A:col10/1732084871849/Put/seqid=0 2024-11-20T06:41:12,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742134_1310 (size=9857) 2024-11-20T06:41:12,760 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/f612c67d31d643e199c834f2c49a2690 2024-11-20T06:41:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/a645df4f535944d99eb8bf6ee3635a69 is 50, key is test_row_0/B:col10/1732084871849/Put/seqid=0 2024-11-20T06:41:12,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742135_1311 (size=9857) 2024-11-20T06:41:12,774 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/a645df4f535944d99eb8bf6ee3635a69 2024-11-20T06:41:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/82e8a739d3eb41bf819426c0a322c6fb is 50, key is test_row_0/C:col10/1732084871849/Put/seqid=0 2024-11-20T06:41:12,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742136_1312 (size=9857) 2024-11-20T06:41:12,789 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/82e8a739d3eb41bf819426c0a322c6fb 2024-11-20T06:41:12,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/f612c67d31d643e199c834f2c49a2690 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f612c67d31d643e199c834f2c49a2690 2024-11-20T06:41:12,797 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f612c67d31d643e199c834f2c49a2690, entries=100, sequenceid=393, filesize=9.6 K 2024-11-20T06:41:12,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/a645df4f535944d99eb8bf6ee3635a69 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a645df4f535944d99eb8bf6ee3635a69 2024-11-20T06:41:12,805 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a645df4f535944d99eb8bf6ee3635a69, entries=100, sequenceid=393, filesize=9.6 K 2024-11-20T06:41:12,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/82e8a739d3eb41bf819426c0a322c6fb as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/82e8a739d3eb41bf819426c0a322c6fb 2024-11-20T06:41:12,810 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/82e8a739d3eb41bf819426c0a322c6fb, entries=100, sequenceid=393, filesize=9.6 K 2024-11-20T06:41:12,811 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 792921c2784cd0f6a5ba47b7f4462f5c in 62ms, sequenceid=393, compaction requested=false 2024-11-20T06:41:12,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:12,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:12,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-20T06:41:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-20T06:41:12,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-20T06:41:12,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 825 msec 2024-11-20T06:41:12,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 831 msec 2024-11-20T06:41:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:12,991 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:12,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:41:12,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/4a455585010048348741b62154c0065c is 50, key is test_row_0/A:col10/1732084872988/Put/seqid=0 2024-11-20T06:41:12,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742137_1313 (size=14741) 2024-11-20T06:41:12,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/4a455585010048348741b62154c0065c 2024-11-20T06:41:13,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/354f2b40a4e949e68bcb2017441a901b is 50, key is test_row_0/B:col10/1732084872988/Put/seqid=0 2024-11-20T06:41:13,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742138_1314 (size=12301) 2024-11-20T06:41:13,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084933017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084933018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084933021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084933022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,102 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/5b567b9df3724a92905326cbcf0f5df0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/5b567b9df3724a92905326cbcf0f5df0 2024-11-20T06:41:13,107 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into 5b567b9df3724a92905326cbcf0f5df0(size=12.9 K), total size for store is 22.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:13,107 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:13,107 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084872686; duration=0sec 2024-11-20T06:41:13,107 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:13,107 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:13,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T06:41:13,124 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:13,124 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-20T06:41:13,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T06:41:13,126 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:13,127 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:13,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:13,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084933124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084933134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084933134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084933134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T06:41:13,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T06:41:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084933334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084933344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084933344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084933344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/354f2b40a4e949e68bcb2017441a901b 2024-11-20T06:41:13,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/95fd8fddda6748259e284fb274d34ddf is 50, key is test_row_0/C:col10/1732084872988/Put/seqid=0 2024-11-20T06:41:13,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742139_1315 (size=12301) 2024-11-20T06:41:13,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T06:41:13,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:13,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T06:41:13,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T06:41:13,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:13,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084933644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084933654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084933654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084933654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:13,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:13,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T06:41:13,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:13,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:13,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T06:41:13,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/95fd8fddda6748259e284fb274d34ddf 2024-11-20T06:41:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/4a455585010048348741b62154c0065c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4a455585010048348741b62154c0065c 2024-11-20T06:41:13,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4a455585010048348741b62154c0065c, entries=200, sequenceid=404, filesize=14.4 K 2024-11-20T06:41:13,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/354f2b40a4e949e68bcb2017441a901b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/354f2b40a4e949e68bcb2017441a901b 2024-11-20T06:41:13,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/354f2b40a4e949e68bcb2017441a901b, entries=150, sequenceid=404, filesize=12.0 K 2024-11-20T06:41:13,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/95fd8fddda6748259e284fb274d34ddf as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95fd8fddda6748259e284fb274d34ddf 2024-11-20T06:41:13,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95fd8fddda6748259e284fb274d34ddf, entries=150, sequenceid=404, filesize=12.0 K 2024-11-20T06:41:13,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 792921c2784cd0f6a5ba47b7f4462f5c in 858ms, sequenceid=404, compaction requested=true 2024-11-20T06:41:13,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:13,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:13,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:13,850 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:13,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:13,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:13,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:13,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:13,850 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:13,851 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37819 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:13,851 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:13,851 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,851 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:13,851 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/46a69f6e9cd246cc95a18dd978837fce, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f612c67d31d643e199c834f2c49a2690, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4a455585010048348741b62154c0065c] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=36.9 K 2024-11-20T06:41:13,851 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:13,851 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,851 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/5b567b9df3724a92905326cbcf0f5df0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a645df4f535944d99eb8bf6ee3635a69, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/354f2b40a4e949e68bcb2017441a901b] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=34.5 K 2024-11-20T06:41:13,851 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46a69f6e9cd246cc95a18dd978837fce, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732084870697 2024-11-20T06:41:13,851 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b567b9df3724a92905326cbcf0f5df0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732084870697 2024-11-20T06:41:13,851 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting f612c67d31d643e199c834f2c49a2690, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732084871846 2024-11-20T06:41:13,852 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a645df4f535944d99eb8bf6ee3635a69, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732084871846 2024-11-20T06:41:13,852 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a455585010048348741b62154c0065c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732084872986 2024-11-20T06:41:13,852 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 354f2b40a4e949e68bcb2017441a901b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732084872987 2024-11-20T06:41:13,860 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:13,860 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:13,861 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/6c27addbb5844cc28727d3e0dcb75a23 is 50, key is test_row_0/A:col10/1732084872988/Put/seqid=0 2024-11-20T06:41:13,861 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/fd35200029764fdc94a521e06cb8ebd5 is 50, key is test_row_0/B:col10/1732084872988/Put/seqid=0 2024-11-20T06:41:13,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742141_1317 (size=13323) 2024-11-20T06:41:13,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742140_1316 (size=13323) 2024-11-20T06:41:13,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T06:41:13,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:13,891 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T06:41:13,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/3d568c75bae34711aa860e1d8b7bff3b is 50, key is test_row_0/A:col10/1732084873017/Put/seqid=0 2024-11-20T06:41:13,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742142_1318 (size=12301) 2024-11-20T06:41:14,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:14,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084934168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084934169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084934171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084934172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T06:41:14,268 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/fd35200029764fdc94a521e06cb8ebd5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/fd35200029764fdc94a521e06cb8ebd5 2024-11-20T06:41:14,268 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/6c27addbb5844cc28727d3e0dcb75a23 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6c27addbb5844cc28727d3e0dcb75a23 2024-11-20T06:41:14,272 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into 6c27addbb5844cc28727d3e0dcb75a23(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:14,272 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into fd35200029764fdc94a521e06cb8ebd5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:14,272 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:14,272 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:14,272 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084873850; duration=0sec 2024-11-20T06:41:14,272 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084873850; duration=0sec 2024-11-20T06:41:14,272 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:14,272 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:14,272 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:14,272 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:14,272 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:14,273 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:14,273 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:14,273 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:14,274 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ca0ae88bfa3644dcb18a17ab3cabdc61, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/82e8a739d3eb41bf819426c0a322c6fb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95fd8fddda6748259e284fb274d34ddf] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=34.5 K 2024-11-20T06:41:14,274 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca0ae88bfa3644dcb18a17ab3cabdc61, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732084870697 2024-11-20T06:41:14,274 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e8a739d3eb41bf819426c0a322c6fb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732084871846 2024-11-20T06:41:14,274 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95fd8fddda6748259e284fb274d34ddf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732084872987 2024-11-20T06:41:14,279 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:14,280 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/18406014a0614a2aae5cba694fd39e99 is 50, key is test_row_0/C:col10/1732084872988/Put/seqid=0 2024-11-20T06:41:14,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742143_1319 (size=13323) 2024-11-20T06:41:14,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084934284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084934284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084934284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084934284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,298 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/3d568c75bae34711aa860e1d8b7bff3b 2024-11-20T06:41:14,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/7f8c9507c34644c9912b4f569e42bdd9 is 50, key is test_row_0/B:col10/1732084873017/Put/seqid=0 2024-11-20T06:41:14,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742144_1320 (size=12301) 2024-11-20T06:41:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084934494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084934494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084934494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084934494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,686 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/18406014a0614a2aae5cba694fd39e99 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/18406014a0614a2aae5cba694fd39e99 2024-11-20T06:41:14,691 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into 18406014a0614a2aae5cba694fd39e99(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:14,691 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:14,691 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084873850; duration=0sec 2024-11-20T06:41:14,691 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:14,691 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:14,710 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/7f8c9507c34644c9912b4f569e42bdd9 2024-11-20T06:41:14,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/8939d1239cf14ff0a3356ac434414d37 is 50, key is test_row_0/C:col10/1732084873017/Put/seqid=0 2024-11-20T06:41:14,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742145_1321 (size=12301) 2024-11-20T06:41:14,722 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/8939d1239cf14ff0a3356ac434414d37 2024-11-20T06:41:14,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/3d568c75bae34711aa860e1d8b7bff3b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3d568c75bae34711aa860e1d8b7bff3b 2024-11-20T06:41:14,730 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3d568c75bae34711aa860e1d8b7bff3b, entries=150, sequenceid=430, filesize=12.0 K 2024-11-20T06:41:14,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/7f8c9507c34644c9912b4f569e42bdd9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/7f8c9507c34644c9912b4f569e42bdd9 2024-11-20T06:41:14,735 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/7f8c9507c34644c9912b4f569e42bdd9, entries=150, sequenceid=430, filesize=12.0 K 2024-11-20T06:41:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/8939d1239cf14ff0a3356ac434414d37 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8939d1239cf14ff0a3356ac434414d37 2024-11-20T06:41:14,740 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8939d1239cf14ff0a3356ac434414d37, entries=150, sequenceid=430, filesize=12.0 K 2024-11-20T06:41:14,741 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 792921c2784cd0f6a5ba47b7f4462f5c in 850ms, sequenceid=430, compaction requested=false 2024-11-20T06:41:14,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:14,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:14,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-20T06:41:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-20T06:41:14,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-20T06:41:14,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6150 sec 2024-11-20T06:41:14,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.6200 sec 2024-11-20T06:41:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:14,805 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:14,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T06:41:14,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/719d38379f2d4eb19782a3f824997687 is 50, key is test_row_0/A:col10/1732084874170/Put/seqid=0 2024-11-20T06:41:14,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742146_1322 (size=12301) 2024-11-20T06:41:14,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084934830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084934830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084934832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084934833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084934944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084934944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084934944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084934944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:15,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084935154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:15,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084935154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084935154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084935154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:15,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/719d38379f2d4eb19782a3f824997687 2024-11-20T06:41:15,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/bc4086f21b4b4fc8ab64e43742036047 is 50, key is test_row_0/B:col10/1732084874170/Put/seqid=0 2024-11-20T06:41:15,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742147_1323 (size=12301) 2024-11-20T06:41:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T06:41:15,274 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:15,274 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-20T06:41:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T06:41:15,276 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:15,277 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:15,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T06:41:15,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T06:41:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:15,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:15,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084935463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084935463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084935464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:15,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084935464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:15,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T06:41:15,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:15,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:15,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:15,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:15,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T06:41:15,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/bc4086f21b4b4fc8ab64e43742036047 2024-11-20T06:41:15,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/c911a07d56dc41a7aed466b0b617882e is 50, key is test_row_0/C:col10/1732084874170/Put/seqid=0 2024-11-20T06:41:15,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742148_1324 (size=12301) 2024-11-20T06:41:15,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/c911a07d56dc41a7aed466b0b617882e 2024-11-20T06:41:15,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/719d38379f2d4eb19782a3f824997687 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/719d38379f2d4eb19782a3f824997687 2024-11-20T06:41:15,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/719d38379f2d4eb19782a3f824997687, entries=150, sequenceid=447, filesize=12.0 K 2024-11-20T06:41:15,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/bc4086f21b4b4fc8ab64e43742036047 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/bc4086f21b4b4fc8ab64e43742036047 2024-11-20T06:41:15,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/bc4086f21b4b4fc8ab64e43742036047, entries=150, sequenceid=447, filesize=12.0 K 2024-11-20T06:41:15,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/c911a07d56dc41a7aed466b0b617882e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c911a07d56dc41a7aed466b0b617882e 2024-11-20T06:41:15,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c911a07d56dc41a7aed466b0b617882e, entries=150, sequenceid=447, filesize=12.0 K 2024-11-20T06:41:15,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 792921c2784cd0f6a5ba47b7f4462f5c in 884ms, sequenceid=447, compaction requested=true 2024-11-20T06:41:15,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:15,690 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:15,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:15,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:15,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:15,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:15,690 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:15,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:15,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:15,691 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:15,691 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:15,691 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:15,691 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6c27addbb5844cc28727d3e0dcb75a23, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3d568c75bae34711aa860e1d8b7bff3b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/719d38379f2d4eb19782a3f824997687] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=37.0 K 2024-11-20T06:41:15,691 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:15,691 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:15,692 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:15,692 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/fd35200029764fdc94a521e06cb8ebd5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/7f8c9507c34644c9912b4f569e42bdd9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/bc4086f21b4b4fc8ab64e43742036047] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=37.0 K 2024-11-20T06:41:15,692 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c27addbb5844cc28727d3e0dcb75a23, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732084872987 2024-11-20T06:41:15,693 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting fd35200029764fdc94a521e06cb8ebd5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732084872987 2024-11-20T06:41:15,693 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d568c75bae34711aa860e1d8b7bff3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732084873017 2024-11-20T06:41:15,693 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 719d38379f2d4eb19782a3f824997687, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732084874170 2024-11-20T06:41:15,694 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8c9507c34644c9912b4f569e42bdd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732084873017 2024-11-20T06:41:15,696 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting bc4086f21b4b4fc8ab64e43742036047, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732084874170 2024-11-20T06:41:15,710 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:15,711 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/bd0bd3ae861e4b0aa098062ba4b1c9b4 is 50, key is test_row_0/A:col10/1732084874170/Put/seqid=0 2024-11-20T06:41:15,713 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:15,713 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/5a27323c8eb441159796368ca44961f0 is 50, key is test_row_0/B:col10/1732084874170/Put/seqid=0 2024-11-20T06:41:15,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742149_1325 (size=13425) 2024-11-20T06:41:15,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742150_1326 (size=13425) 2024-11-20T06:41:15,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T06:41:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:15,736 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T06:41:15,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/463173b0785144e28d4738ce1ecf6d01 is 50, key is test_row_0/A:col10/1732084874823/Put/seqid=0 2024-11-20T06:41:15,743 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/5a27323c8eb441159796368ca44961f0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/5a27323c8eb441159796368ca44961f0 2024-11-20T06:41:15,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742151_1327 (size=12301) 2024-11-20T06:41:15,748 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into 5a27323c8eb441159796368ca44961f0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:15,748 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:15,748 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084875690; duration=0sec 2024-11-20T06:41:15,748 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:15,748 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:15,749 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:15,751 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/463173b0785144e28d4738ce1ecf6d01 2024-11-20T06:41:15,752 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:15,752 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:15,752 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:15,752 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/18406014a0614a2aae5cba694fd39e99, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8939d1239cf14ff0a3356ac434414d37, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c911a07d56dc41a7aed466b0b617882e] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=37.0 K 2024-11-20T06:41:15,753 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 18406014a0614a2aae5cba694fd39e99, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732084872987 2024-11-20T06:41:15,754 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8939d1239cf14ff0a3356ac434414d37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732084873017 2024-11-20T06:41:15,754 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c911a07d56dc41a7aed466b0b617882e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732084874170 2024-11-20T06:41:15,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/17927b10b91f47b9a295d54be087e797 is 50, key is test_row_0/B:col10/1732084874823/Put/seqid=0 2024-11-20T06:41:15,780 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:15,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742152_1328 (size=12301) 2024-11-20T06:41:15,781 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/a52005c66da043d1bc5481c6c9a3ea04 is 50, key is test_row_0/C:col10/1732084874170/Put/seqid=0 2024-11-20T06:41:15,784 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/17927b10b91f47b9a295d54be087e797 2024-11-20T06:41:15,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/7354c0e3eadd4a2985b64fc6395d813e is 50, key is test_row_0/C:col10/1732084874823/Put/seqid=0 2024-11-20T06:41:15,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742153_1329 (size=13425) 2024-11-20T06:41:15,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742154_1330 (size=12301) 2024-11-20T06:41:15,803 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/7354c0e3eadd4a2985b64fc6395d813e 2024-11-20T06:41:15,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/463173b0785144e28d4738ce1ecf6d01 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/463173b0785144e28d4738ce1ecf6d01 2024-11-20T06:41:15,807 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/a52005c66da043d1bc5481c6c9a3ea04 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/a52005c66da043d1bc5481c6c9a3ea04 2024-11-20T06:41:15,810 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/463173b0785144e28d4738ce1ecf6d01, entries=150, sequenceid=469, filesize=12.0 K 2024-11-20T06:41:15,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/17927b10b91f47b9a295d54be087e797 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/17927b10b91f47b9a295d54be087e797 2024-11-20T06:41:15,814 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/17927b10b91f47b9a295d54be087e797, entries=150, sequenceid=469, filesize=12.0 K 2024-11-20T06:41:15,815 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into a52005c66da043d1bc5481c6c9a3ea04(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:15,815 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:15,815 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084875690; duration=0sec 2024-11-20T06:41:15,815 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:15,815 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:15,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/7354c0e3eadd4a2985b64fc6395d813e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/7354c0e3eadd4a2985b64fc6395d813e 2024-11-20T06:41:15,819 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/7354c0e3eadd4a2985b64fc6395d813e, entries=150, sequenceid=469, filesize=12.0 K 2024-11-20T06:41:15,820 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 792921c2784cd0f6a5ba47b7f4462f5c in 83ms, sequenceid=469, compaction requested=false 2024-11-20T06:41:15,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:15,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:15,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-20T06:41:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-20T06:41:15,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-20T06:41:15,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 544 msec 2024-11-20T06:41:15,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 550 msec 2024-11-20T06:41:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T06:41:15,904 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:15,905 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-20T06:41:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T06:41:15,907 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:15,908 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:15,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:15,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:15,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T06:41:15,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/2429da4424da445690b0d6be6d285d2e is 50, key is test_row_0/A:col10/1732084875979/Put/seqid=0 2024-11-20T06:41:16,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742155_1331 (size=17181) 2024-11-20T06:41:16,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084936009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084936010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T06:41:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084936010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084936012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T06:41:16,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:16,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,075 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-11-20T06:41:16,075 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:16,075 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:16,075 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:16,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:16,077 INFO [Thread-1065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1065. 2024-11-20T06:41:16,077 DEBUG [Thread-1065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:16,077 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:16,078 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:16,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:16,080 INFO [Thread-1067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1067. 2024-11-20T06:41:16,080 DEBUG [Thread-1067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:16,080 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:16,080 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:16,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:16,081 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-11-20T06:41:16,081 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:16,081 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:16,081 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:16,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:16,081 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-11-20T06:41:16,081 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:16,081 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:16,081 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:16,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:16,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084936123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084936123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084936124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084936124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,126 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/bd0bd3ae861e4b0aa098062ba4b1c9b4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bd0bd3ae861e4b0aa098062ba4b1c9b4 2024-11-20T06:41:16,131 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into bd0bd3ae861e4b0aa098062ba4b1c9b4(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:16,131 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:16,131 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084875690; duration=0sec 2024-11-20T06:41:16,131 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:16,131 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:16,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T06:41:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T06:41:16,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084936334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084936334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084936334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084936334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T06:41:16,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:16,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/2429da4424da445690b0d6be6d285d2e 2024-11-20T06:41:16,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/8cb6d9e393d04353b93a056140759c79 is 50, key is test_row_0/B:col10/1732084875979/Put/seqid=0 2024-11-20T06:41:16,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742156_1332 (size=12301) 2024-11-20T06:41:16,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T06:41:16,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T06:41:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084936644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084936644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084936644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084936645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:16,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T06:41:16,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:16,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/8cb6d9e393d04353b93a056140759c79 2024-11-20T06:41:16,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ef89eac5ffd349b28023ca52263e9f17 is 50, key is test_row_0/C:col10/1732084875979/Put/seqid=0 2024-11-20T06:41:16,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742157_1333 (size=12301) 2024-11-20T06:41:16,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T06:41:16,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:16,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T06:41:16,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:16,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:16,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T06:41:17,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T06:41:17,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:17,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. as already flushing 2024-11-20T06:41:17,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:17,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:17,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:17,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:17,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:17,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45130 deadline: 1732084937154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45090 deadline: 1732084937154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45162 deadline: 1732084937154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:17,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45136 deadline: 1732084937155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:17,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ef89eac5ffd349b28023ca52263e9f17 2024-11-20T06:41:17,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/2429da4424da445690b0d6be6d285d2e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/2429da4424da445690b0d6be6d285d2e 2024-11-20T06:41:17,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/2429da4424da445690b0d6be6d285d2e, entries=250, sequenceid=483, filesize=16.8 K 2024-11-20T06:41:17,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/8cb6d9e393d04353b93a056140759c79 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8cb6d9e393d04353b93a056140759c79 2024-11-20T06:41:17,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8cb6d9e393d04353b93a056140759c79, entries=150, sequenceid=483, filesize=12.0 K 2024-11-20T06:41:17,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/ef89eac5ffd349b28023ca52263e9f17 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ef89eac5ffd349b28023ca52263e9f17 2024-11-20T06:41:17,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ef89eac5ffd349b28023ca52263e9f17, entries=150, sequenceid=483, filesize=12.0 K 2024-11-20T06:41:17,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 792921c2784cd0f6a5ba47b7f4462f5c in 1267ms, sequenceid=483, compaction requested=true 2024-11-20T06:41:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:17,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:17,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:17,248 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:17,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:17,248 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:17,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 792921c2784cd0f6a5ba47b7f4462f5c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:17,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:17,248 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:17,248 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:17,248 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/A is initiating minor compaction (all files) 2024-11-20T06:41:17,248 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/B is initiating minor compaction (all files) 2024-11-20T06:41:17,248 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/A in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:17,248 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/B in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:17,249 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bd0bd3ae861e4b0aa098062ba4b1c9b4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/463173b0785144e28d4738ce1ecf6d01, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/2429da4424da445690b0d6be6d285d2e] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=41.9 K 2024-11-20T06:41:17,249 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/5a27323c8eb441159796368ca44961f0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/17927b10b91f47b9a295d54be087e797, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8cb6d9e393d04353b93a056140759c79] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=37.1 K 2024-11-20T06:41:17,249 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd0bd3ae861e4b0aa098062ba4b1c9b4, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732084874170 2024-11-20T06:41:17,249 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a27323c8eb441159796368ca44961f0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732084874170 2024-11-20T06:41:17,249 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 463173b0785144e28d4738ce1ecf6d01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732084874823 2024-11-20T06:41:17,249 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 17927b10b91f47b9a295d54be087e797, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732084874823 2024-11-20T06:41:17,249 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2429da4424da445690b0d6be6d285d2e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732084875974 2024-11-20T06:41:17,249 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb6d9e393d04353b93a056140759c79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732084875978 2024-11-20T06:41:17,259 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:17,259 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/457bf5b9d8d34855bfe3090dee9a162b is 50, key is test_row_0/B:col10/1732084875979/Put/seqid=0 2024-11-20T06:41:17,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742158_1334 (size=13527) 2024-11-20T06:41:17,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T06:41:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:17,294 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T06:41:17,295 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#A#compaction#295 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:17,297 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/af860f7b5458460c86eaf7e15267d560 is 50, key is test_row_0/A:col10/1732084875979/Put/seqid=0 2024-11-20T06:41:17,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742159_1335 (size=13527) 2024-11-20T06:41:17,308 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/af860f7b5458460c86eaf7e15267d560 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/af860f7b5458460c86eaf7e15267d560 2024-11-20T06:41:17,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/13e0765a1c5647b6a968f30b59befb61 is 50, key is test_row_0/A:col10/1732084876008/Put/seqid=0 2024-11-20T06:41:17,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742160_1336 (size=12301) 2024-11-20T06:41:17,313 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/A of 792921c2784cd0f6a5ba47b7f4462f5c into af860f7b5458460c86eaf7e15267d560(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:17,313 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:17,313 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/A, priority=13, startTime=1732084877247; duration=0sec 2024-11-20T06:41:17,313 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:17,313 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:A 2024-11-20T06:41:17,313 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:17,314 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:17,314 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 792921c2784cd0f6a5ba47b7f4462f5c/C is initiating minor compaction (all files) 2024-11-20T06:41:17,314 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 792921c2784cd0f6a5ba47b7f4462f5c/C in TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:17,314 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/a52005c66da043d1bc5481c6c9a3ea04, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/7354c0e3eadd4a2985b64fc6395d813e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ef89eac5ffd349b28023ca52263e9f17] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp, totalSize=37.1 K 2024-11-20T06:41:17,314 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting a52005c66da043d1bc5481c6c9a3ea04, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732084874170 2024-11-20T06:41:17,314 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7354c0e3eadd4a2985b64fc6395d813e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732084874823 2024-11-20T06:41:17,314 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef89eac5ffd349b28023ca52263e9f17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732084875978 2024-11-20T06:41:17,326 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 792921c2784cd0f6a5ba47b7f4462f5c#C#compaction#297 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:17,326 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/689b3ff144754debb50bab45db153463 is 50, key is test_row_0/C:col10/1732084875979/Put/seqid=0 2024-11-20T06:41:17,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742161_1337 (size=13527) 2024-11-20T06:41:17,686 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/457bf5b9d8d34855bfe3090dee9a162b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/457bf5b9d8d34855bfe3090dee9a162b 2024-11-20T06:41:17,690 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/B of 792921c2784cd0f6a5ba47b7f4462f5c into 457bf5b9d8d34855bfe3090dee9a162b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:17,690 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:17,690 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/B, priority=13, startTime=1732084877247; duration=0sec 2024-11-20T06:41:17,690 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:17,690 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:B 2024-11-20T06:41:17,713 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/13e0765a1c5647b6a968f30b59befb61 2024-11-20T06:41:17,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/336300e3226c4613b223e5f29c170ea5 is 50, key is test_row_0/B:col10/1732084876008/Put/seqid=0 2024-11-20T06:41:17,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742162_1338 (size=12301) 2024-11-20T06:41:17,723 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/336300e3226c4613b223e5f29c170ea5 2024-11-20T06:41:17,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/0da6ccce3ac84a98a15a44ccd4068e24 is 50, key is test_row_0/C:col10/1732084876008/Put/seqid=0 2024-11-20T06:41:17,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742163_1339 (size=12301) 2024-11-20T06:41:17,743 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/689b3ff144754debb50bab45db153463 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/689b3ff144754debb50bab45db153463 2024-11-20T06:41:17,748 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 792921c2784cd0f6a5ba47b7f4462f5c/C of 792921c2784cd0f6a5ba47b7f4462f5c into 689b3ff144754debb50bab45db153463(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:17,748 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:17,748 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., storeName=792921c2784cd0f6a5ba47b7f4462f5c/C, priority=13, startTime=1732084877248; duration=0sec 2024-11-20T06:41:17,748 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:17,748 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 792921c2784cd0f6a5ba47b7f4462f5c:C 2024-11-20T06:41:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T06:41:18,134 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/0da6ccce3ac84a98a15a44ccd4068e24 2024-11-20T06:41:18,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/13e0765a1c5647b6a968f30b59befb61 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/13e0765a1c5647b6a968f30b59befb61 2024-11-20T06:41:18,144 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/13e0765a1c5647b6a968f30b59befb61, entries=150, sequenceid=508, filesize=12.0 K 2024-11-20T06:41:18,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/336300e3226c4613b223e5f29c170ea5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/336300e3226c4613b223e5f29c170ea5 2024-11-20T06:41:18,149 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/336300e3226c4613b223e5f29c170ea5, entries=150, sequenceid=508, filesize=12.0 K 2024-11-20T06:41:18,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/0da6ccce3ac84a98a15a44ccd4068e24 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/0da6ccce3ac84a98a15a44ccd4068e24 2024-11-20T06:41:18,153 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/0da6ccce3ac84a98a15a44ccd4068e24, entries=150, sequenceid=508, filesize=12.0 K 2024-11-20T06:41:18,154 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 792921c2784cd0f6a5ba47b7f4462f5c in 860ms, sequenceid=508, compaction requested=false 2024-11-20T06:41:18,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 792921c2784cd0f6a5ba47b7f4462f5c: 2024-11-20T06:41:18,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:18,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-20T06:41:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-20T06:41:18,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-20T06:41:18,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2480 sec 2024-11-20T06:41:18,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.2530 sec 2024-11-20T06:41:18,166 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-20T06:41:18,166 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-11-20T06:41:18,167 INFO [Thread-1062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1062. 2024-11-20T06:41:18,167 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:18,167 DEBUG [Thread-1062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:18,167 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:18,167 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:18,167 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:18,167 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:18,167 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:18,167 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:18,167 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:18,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:18,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:18,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:18,168 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-11-20T06:41:18,168 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:18,168 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:18,168 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:18,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T06:41:20,064 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:22,025 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-20T06:41:22,025 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:22,025 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:22,025 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:22,025 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4272 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4294 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4236 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4257 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4282 2024-11-20T06:41:22,026 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T06:41:22,026 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:41:22,026 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:41:22,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:22,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:22,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:22,027 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T06:41:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T06:41:22,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084882031"}]},"ts":"1732084882031"} 2024-11-20T06:41:22,033 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T06:41:22,033 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T06:41:22,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T06:41:22,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=792921c2784cd0f6a5ba47b7f4462f5c, UNASSIGN}] 2024-11-20T06:41:22,036 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=792921c2784cd0f6a5ba47b7f4462f5c, UNASSIGN 2024-11-20T06:41:22,036 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=792921c2784cd0f6a5ba47b7f4462f5c, regionState=CLOSING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:22,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=792921c2784cd0f6a5ba47b7f4462f5c, UNASSIGN because future has completed 2024-11-20T06:41:22,038 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T06:41:22,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:41:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T06:41:22,191 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:22,191 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T06:41:22,191 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 792921c2784cd0f6a5ba47b7f4462f5c, disabling compactions & flushes 2024-11-20T06:41:22,191 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:22,191 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:22,191 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. after waiting 0 ms 2024-11-20T06:41:22,191 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:22,191 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 792921c2784cd0f6a5ba47b7f4462f5c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T06:41:22,195 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/7645bef31568449f8457a68aaa6fce10 is 50, key is test_row_0/A:col10/1732084878166/Put/seqid=0 2024-11-20T06:41:22,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742164_1340 (size=12301) 2024-11-20T06:41:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T06:41:22,599 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/7645bef31568449f8457a68aaa6fce10 2024-11-20T06:41:22,605 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/8ee81ffdc4944ead8be5d47266fce50b is 50, key is test_row_0/B:col10/1732084878166/Put/seqid=0 2024-11-20T06:41:22,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742165_1341 (size=12301) 2024-11-20T06:41:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T06:41:22,878 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T06:41:23,012 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/8ee81ffdc4944ead8be5d47266fce50b 2024-11-20T06:41:23,018 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/47b0f6781b274d0281b7cf0eec51d1b6 is 50, key is test_row_0/C:col10/1732084878166/Put/seqid=0 2024-11-20T06:41:23,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742166_1342 (size=12301) 2024-11-20T06:41:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T06:41:23,422 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/47b0f6781b274d0281b7cf0eec51d1b6 2024-11-20T06:41:23,427 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/A/7645bef31568449f8457a68aaa6fce10 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/7645bef31568449f8457a68aaa6fce10 2024-11-20T06:41:23,431 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/7645bef31568449f8457a68aaa6fce10, entries=150, sequenceid=519, filesize=12.0 K 2024-11-20T06:41:23,431 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/B/8ee81ffdc4944ead8be5d47266fce50b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8ee81ffdc4944ead8be5d47266fce50b 2024-11-20T06:41:23,435 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8ee81ffdc4944ead8be5d47266fce50b, entries=150, sequenceid=519, filesize=12.0 K 2024-11-20T06:41:23,436 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/.tmp/C/47b0f6781b274d0281b7cf0eec51d1b6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/47b0f6781b274d0281b7cf0eec51d1b6 2024-11-20T06:41:23,440 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/47b0f6781b274d0281b7cf0eec51d1b6, entries=150, sequenceid=519, filesize=12.0 K 2024-11-20T06:41:23,441 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 792921c2784cd0f6a5ba47b7f4462f5c in 1250ms, sequenceid=519, compaction requested=true 2024-11-20T06:41:23,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f825710b29e042648a9b2ed7151344d9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bdb21a5513a14d769f6e5414dfa69ffb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/5014baa2409f4162b01fa662b3f0bd1f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c1119381facb40c29ccc40fd8512ca4d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f5d20f6aff5040dd9b2a73a72df51c3f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/39de4f93bc56433389a3e2fd67d6542b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c6eabad16660451ca1bff5aeb2432ae0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6a84fda3aaa04b1aa7d55630eeddcb39, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/803b9577184544579022d386434c6b18, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/9c553eb782d84096af9be98153a8d51a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/8b16e62437a941f39256cf24c326298a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/624f2bd573bc44b19e5302c07030b4a2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/972ce1f63885453a900e8079e89b3348, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/807778a9b31644dbb5d8706963bd8aa2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3e479e96785f4f92b870431ed304990e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4aa3ce84b9e4425b91a8344c005263a3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/0c18371a5da94cd8a9b9fe1ff719ce71, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c4371b6897f1436a9cb356d9dbcc9ff5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/659477d78152413abfd962baf398a89b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c21dcd706eff47039f21d11875a840a2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/9b5f7575974d49afabac0850b28ae7f0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/7c84d8b6588f4cda89bc0629f40f7274, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/d53253f126e54d40993690f0ea985097, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/06bf45f724b84eb48c5c162e8661369e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/b6ae5561942742308e25b1ba302a4e8a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3b8b2ccba9c5409eb58e0d72d99051ab, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/33227c761a3148ffbe434ccd737ebc15, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/46a69f6e9cd246cc95a18dd978837fce, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f612c67d31d643e199c834f2c49a2690, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4a455585010048348741b62154c0065c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6c27addbb5844cc28727d3e0dcb75a23, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3d568c75bae34711aa860e1d8b7bff3b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bd0bd3ae861e4b0aa098062ba4b1c9b4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/719d38379f2d4eb19782a3f824997687, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/463173b0785144e28d4738ce1ecf6d01, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/2429da4424da445690b0d6be6d285d2e] to archive 2024-11-20T06:41:23,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:41:23,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f825710b29e042648a9b2ed7151344d9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f825710b29e042648a9b2ed7151344d9 2024-11-20T06:41:23,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bdb21a5513a14d769f6e5414dfa69ffb to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bdb21a5513a14d769f6e5414dfa69ffb 2024-11-20T06:41:23,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/5014baa2409f4162b01fa662b3f0bd1f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/5014baa2409f4162b01fa662b3f0bd1f 2024-11-20T06:41:23,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c1119381facb40c29ccc40fd8512ca4d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c1119381facb40c29ccc40fd8512ca4d 2024-11-20T06:41:23,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f5d20f6aff5040dd9b2a73a72df51c3f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f5d20f6aff5040dd9b2a73a72df51c3f 2024-11-20T06:41:23,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/39de4f93bc56433389a3e2fd67d6542b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/39de4f93bc56433389a3e2fd67d6542b 2024-11-20T06:41:23,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c6eabad16660451ca1bff5aeb2432ae0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c6eabad16660451ca1bff5aeb2432ae0 2024-11-20T06:41:23,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6a84fda3aaa04b1aa7d55630eeddcb39 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6a84fda3aaa04b1aa7d55630eeddcb39 2024-11-20T06:41:23,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/803b9577184544579022d386434c6b18 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/803b9577184544579022d386434c6b18 2024-11-20T06:41:23,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/9c553eb782d84096af9be98153a8d51a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/9c553eb782d84096af9be98153a8d51a 2024-11-20T06:41:23,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/8b16e62437a941f39256cf24c326298a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/8b16e62437a941f39256cf24c326298a 2024-11-20T06:41:23,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/624f2bd573bc44b19e5302c07030b4a2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/624f2bd573bc44b19e5302c07030b4a2 2024-11-20T06:41:23,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/972ce1f63885453a900e8079e89b3348 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/972ce1f63885453a900e8079e89b3348 2024-11-20T06:41:23,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/807778a9b31644dbb5d8706963bd8aa2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/807778a9b31644dbb5d8706963bd8aa2 2024-11-20T06:41:23,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3e479e96785f4f92b870431ed304990e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3e479e96785f4f92b870431ed304990e 2024-11-20T06:41:23,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4aa3ce84b9e4425b91a8344c005263a3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4aa3ce84b9e4425b91a8344c005263a3 2024-11-20T06:41:23,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/0c18371a5da94cd8a9b9fe1ff719ce71 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/0c18371a5da94cd8a9b9fe1ff719ce71 2024-11-20T06:41:23,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c4371b6897f1436a9cb356d9dbcc9ff5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c4371b6897f1436a9cb356d9dbcc9ff5 2024-11-20T06:41:23,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/659477d78152413abfd962baf398a89b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/659477d78152413abfd962baf398a89b 2024-11-20T06:41:23,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c21dcd706eff47039f21d11875a840a2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/c21dcd706eff47039f21d11875a840a2 2024-11-20T06:41:23,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/9b5f7575974d49afabac0850b28ae7f0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/9b5f7575974d49afabac0850b28ae7f0 2024-11-20T06:41:23,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/7c84d8b6588f4cda89bc0629f40f7274 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/7c84d8b6588f4cda89bc0629f40f7274 2024-11-20T06:41:23,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/d53253f126e54d40993690f0ea985097 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/d53253f126e54d40993690f0ea985097 2024-11-20T06:41:23,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/06bf45f724b84eb48c5c162e8661369e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/06bf45f724b84eb48c5c162e8661369e 2024-11-20T06:41:23,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/b6ae5561942742308e25b1ba302a4e8a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/b6ae5561942742308e25b1ba302a4e8a 2024-11-20T06:41:23,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3b8b2ccba9c5409eb58e0d72d99051ab to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3b8b2ccba9c5409eb58e0d72d99051ab 2024-11-20T06:41:23,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/33227c761a3148ffbe434ccd737ebc15 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/33227c761a3148ffbe434ccd737ebc15 2024-11-20T06:41:23,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/46a69f6e9cd246cc95a18dd978837fce to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/46a69f6e9cd246cc95a18dd978837fce 2024-11-20T06:41:23,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f612c67d31d643e199c834f2c49a2690 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/f612c67d31d643e199c834f2c49a2690 2024-11-20T06:41:23,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4a455585010048348741b62154c0065c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/4a455585010048348741b62154c0065c 2024-11-20T06:41:23,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6c27addbb5844cc28727d3e0dcb75a23 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/6c27addbb5844cc28727d3e0dcb75a23 2024-11-20T06:41:23,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3d568c75bae34711aa860e1d8b7bff3b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/3d568c75bae34711aa860e1d8b7bff3b 2024-11-20T06:41:23,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bd0bd3ae861e4b0aa098062ba4b1c9b4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/bd0bd3ae861e4b0aa098062ba4b1c9b4 2024-11-20T06:41:23,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/719d38379f2d4eb19782a3f824997687 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/719d38379f2d4eb19782a3f824997687 2024-11-20T06:41:23,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/463173b0785144e28d4738ce1ecf6d01 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/463173b0785144e28d4738ce1ecf6d01 2024-11-20T06:41:23,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/2429da4424da445690b0d6be6d285d2e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/2429da4424da445690b0d6be6d285d2e 2024-11-20T06:41:23,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/581cb75591924e57810221793a8b6b51, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b4e172ce28d34131878642326cdbdb35, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a02f9d0e4f5a4e3e8cd3fccc0d8fef2d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0631f4c00fbd4e2b880b5568422a679e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/085bd307591e4008b26b25f99e46f8fe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d9e47c61b8b14819a4838b03e96ed823, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/1243f555e6bb4f46b82ada93332e3ac0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/86d6e1c6f6054c639d078ce81bffbe29, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0c67e0539eaf48858c5c279ec02b6547, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ec7da3fd3819481ca306e7a06cd8a292, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/69803b9c0f404b52aef10d26f4f53975, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/2a6e9246dad24a998f03a2e6936eff9f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b5f665da9bd34b2cb48e0ce1b7d85fb0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/925e74b206bb46fb8f4034f8616356a8, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/3f7940d1536a49649e48ea87870db754, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0ad891d255224a3d9bbfde9c8e762a80, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e6d9949df5ae41c3af369e1a5b268a5e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d721eef1194c44c981a5267fca41a5a5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8b5d64e8e0114667b56ea4ea74d02f89, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/c1b1ec7654154a54b70cb69a6d1af0e4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ad0a55344ae84f3ba7147317813ec7ee, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e8d6c8d3fdcd448a998236e82748edd3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b6fb55f039a74a6fa4bb161a8a6072a2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/9f7c197b354348b4a31c72c92492dc31, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/dacdbe47a3554b32b0a985f50e031b1b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d0cd5980f0e44dc6b2b6614632853e3a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/5b567b9df3724a92905326cbcf0f5df0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e84a63e80eac4924a3a02696a3ce7f4a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a645df4f535944d99eb8bf6ee3635a69, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/fd35200029764fdc94a521e06cb8ebd5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/354f2b40a4e949e68bcb2017441a901b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/7f8c9507c34644c9912b4f569e42bdd9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/5a27323c8eb441159796368ca44961f0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/bc4086f21b4b4fc8ab64e43742036047, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/17927b10b91f47b9a295d54be087e797, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8cb6d9e393d04353b93a056140759c79] to archive 2024-11-20T06:41:23,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:41:23,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/581cb75591924e57810221793a8b6b51 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/581cb75591924e57810221793a8b6b51 2024-11-20T06:41:23,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b4e172ce28d34131878642326cdbdb35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b4e172ce28d34131878642326cdbdb35 2024-11-20T06:41:23,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a02f9d0e4f5a4e3e8cd3fccc0d8fef2d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a02f9d0e4f5a4e3e8cd3fccc0d8fef2d 2024-11-20T06:41:23,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0631f4c00fbd4e2b880b5568422a679e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0631f4c00fbd4e2b880b5568422a679e 2024-11-20T06:41:23,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/085bd307591e4008b26b25f99e46f8fe to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/085bd307591e4008b26b25f99e46f8fe 2024-11-20T06:41:23,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d9e47c61b8b14819a4838b03e96ed823 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d9e47c61b8b14819a4838b03e96ed823 2024-11-20T06:41:23,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/1243f555e6bb4f46b82ada93332e3ac0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/1243f555e6bb4f46b82ada93332e3ac0 2024-11-20T06:41:23,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/86d6e1c6f6054c639d078ce81bffbe29 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/86d6e1c6f6054c639d078ce81bffbe29 2024-11-20T06:41:23,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0c67e0539eaf48858c5c279ec02b6547 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0c67e0539eaf48858c5c279ec02b6547 2024-11-20T06:41:23,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ec7da3fd3819481ca306e7a06cd8a292 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ec7da3fd3819481ca306e7a06cd8a292 2024-11-20T06:41:23,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/69803b9c0f404b52aef10d26f4f53975 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/69803b9c0f404b52aef10d26f4f53975 2024-11-20T06:41:23,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/2a6e9246dad24a998f03a2e6936eff9f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/2a6e9246dad24a998f03a2e6936eff9f 2024-11-20T06:41:23,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b5f665da9bd34b2cb48e0ce1b7d85fb0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b5f665da9bd34b2cb48e0ce1b7d85fb0 2024-11-20T06:41:23,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/925e74b206bb46fb8f4034f8616356a8 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/925e74b206bb46fb8f4034f8616356a8 2024-11-20T06:41:23,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/3f7940d1536a49649e48ea87870db754 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/3f7940d1536a49649e48ea87870db754 2024-11-20T06:41:23,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0ad891d255224a3d9bbfde9c8e762a80 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/0ad891d255224a3d9bbfde9c8e762a80 2024-11-20T06:41:23,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e6d9949df5ae41c3af369e1a5b268a5e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e6d9949df5ae41c3af369e1a5b268a5e 2024-11-20T06:41:23,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d721eef1194c44c981a5267fca41a5a5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d721eef1194c44c981a5267fca41a5a5 2024-11-20T06:41:23,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8b5d64e8e0114667b56ea4ea74d02f89 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8b5d64e8e0114667b56ea4ea74d02f89 2024-11-20T06:41:23,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/c1b1ec7654154a54b70cb69a6d1af0e4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/c1b1ec7654154a54b70cb69a6d1af0e4 2024-11-20T06:41:23,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ad0a55344ae84f3ba7147317813ec7ee to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/ad0a55344ae84f3ba7147317813ec7ee 2024-11-20T06:41:23,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e8d6c8d3fdcd448a998236e82748edd3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e8d6c8d3fdcd448a998236e82748edd3 2024-11-20T06:41:23,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b6fb55f039a74a6fa4bb161a8a6072a2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/b6fb55f039a74a6fa4bb161a8a6072a2 2024-11-20T06:41:23,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/9f7c197b354348b4a31c72c92492dc31 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/9f7c197b354348b4a31c72c92492dc31 2024-11-20T06:41:23,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/dacdbe47a3554b32b0a985f50e031b1b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/dacdbe47a3554b32b0a985f50e031b1b 2024-11-20T06:41:23,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d0cd5980f0e44dc6b2b6614632853e3a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/d0cd5980f0e44dc6b2b6614632853e3a 2024-11-20T06:41:23,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/5b567b9df3724a92905326cbcf0f5df0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/5b567b9df3724a92905326cbcf0f5df0 2024-11-20T06:41:23,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e84a63e80eac4924a3a02696a3ce7f4a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/e84a63e80eac4924a3a02696a3ce7f4a 2024-11-20T06:41:23,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a645df4f535944d99eb8bf6ee3635a69 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/a645df4f535944d99eb8bf6ee3635a69 2024-11-20T06:41:23,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/fd35200029764fdc94a521e06cb8ebd5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/fd35200029764fdc94a521e06cb8ebd5 2024-11-20T06:41:23,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/354f2b40a4e949e68bcb2017441a901b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/354f2b40a4e949e68bcb2017441a901b 2024-11-20T06:41:23,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/7f8c9507c34644c9912b4f569e42bdd9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/7f8c9507c34644c9912b4f569e42bdd9 2024-11-20T06:41:23,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/5a27323c8eb441159796368ca44961f0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/5a27323c8eb441159796368ca44961f0 2024-11-20T06:41:23,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/bc4086f21b4b4fc8ab64e43742036047 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/bc4086f21b4b4fc8ab64e43742036047 2024-11-20T06:41:23,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/17927b10b91f47b9a295d54be087e797 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/17927b10b91f47b9a295d54be087e797 2024-11-20T06:41:23,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8cb6d9e393d04353b93a056140759c79 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8cb6d9e393d04353b93a056140759c79 2024-11-20T06:41:23,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/775382e17f774efd894dc2744e4eae29, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/bc6169fe33464d3cb4b7fa8b1df1c82c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/761f3a32bb9c4a039c14218e234d6c3f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/15636d6b9ba64690a97b17f7a0980ddf, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cb59db1744a34523bab9871c249a4ff5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e476f1c5f2ad48039c2f531275ee1149, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9dc77f229e184209acd629d459694213, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/547cb734a78447b5b13e05c08a7116f2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/75f1f417f6d1460995848fcfcfabef51, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f4076b58af684e04957167f79248a275, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f153acdcf8d946c7a7c5a0717d88ebf0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9b916c011b9d4a51b54e52d6bc4e938b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/4c11955deaf042adbe702c9e5234a2f4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/afbb779a8ebf4276ab86bfd13f425bdb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/66896c426b844d1c96951ebc7e09b56f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/443f03cb7bba467980eb113382953dfc, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95ea88dca63843d09e8ee323ebca2c44, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cc4a6e38de4e4c07a7bead47823e4a60, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad693bb5313540f6989861ec7d65eeb9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c0e12c35ebbf4254a6ace621db99a217, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f0df072cbdaf494ba53929fdf0fcb767, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad4a3f04852541879ebdfd122bb54970, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8f0573648e1340afbd1a8ef1781be35f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e2a095963dd144a2a1ff9f0ae221d53c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/69b6ce941c5b425387d769841c340336, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e482706c11ab479880efbd02db43b979, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ca0ae88bfa3644dcb18a17ab3cabdc61, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/6b103e83f3964fbea64286189147f14b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/82e8a739d3eb41bf819426c0a322c6fb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/18406014a0614a2aae5cba694fd39e99, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95fd8fddda6748259e284fb274d34ddf, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8939d1239cf14ff0a3356ac434414d37, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/a52005c66da043d1bc5481c6c9a3ea04, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c911a07d56dc41a7aed466b0b617882e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/7354c0e3eadd4a2985b64fc6395d813e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ef89eac5ffd349b28023ca52263e9f17] to archive 2024-11-20T06:41:23,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:41:23,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/775382e17f774efd894dc2744e4eae29 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/775382e17f774efd894dc2744e4eae29 2024-11-20T06:41:23,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/bc6169fe33464d3cb4b7fa8b1df1c82c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/bc6169fe33464d3cb4b7fa8b1df1c82c 2024-11-20T06:41:23,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/761f3a32bb9c4a039c14218e234d6c3f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/761f3a32bb9c4a039c14218e234d6c3f 2024-11-20T06:41:23,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/15636d6b9ba64690a97b17f7a0980ddf to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/15636d6b9ba64690a97b17f7a0980ddf 2024-11-20T06:41:23,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cb59db1744a34523bab9871c249a4ff5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cb59db1744a34523bab9871c249a4ff5 2024-11-20T06:41:23,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e476f1c5f2ad48039c2f531275ee1149 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e476f1c5f2ad48039c2f531275ee1149 2024-11-20T06:41:23,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9dc77f229e184209acd629d459694213 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9dc77f229e184209acd629d459694213 2024-11-20T06:41:23,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/547cb734a78447b5b13e05c08a7116f2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/547cb734a78447b5b13e05c08a7116f2 2024-11-20T06:41:23,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/75f1f417f6d1460995848fcfcfabef51 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/75f1f417f6d1460995848fcfcfabef51 2024-11-20T06:41:23,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f4076b58af684e04957167f79248a275 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f4076b58af684e04957167f79248a275 2024-11-20T06:41:23,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f153acdcf8d946c7a7c5a0717d88ebf0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f153acdcf8d946c7a7c5a0717d88ebf0 2024-11-20T06:41:23,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9b916c011b9d4a51b54e52d6bc4e938b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/9b916c011b9d4a51b54e52d6bc4e938b 2024-11-20T06:41:23,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/4c11955deaf042adbe702c9e5234a2f4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/4c11955deaf042adbe702c9e5234a2f4 2024-11-20T06:41:23,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/afbb779a8ebf4276ab86bfd13f425bdb to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/afbb779a8ebf4276ab86bfd13f425bdb 2024-11-20T06:41:23,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/66896c426b844d1c96951ebc7e09b56f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/66896c426b844d1c96951ebc7e09b56f 2024-11-20T06:41:23,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/443f03cb7bba467980eb113382953dfc to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/443f03cb7bba467980eb113382953dfc 2024-11-20T06:41:23,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95ea88dca63843d09e8ee323ebca2c44 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95ea88dca63843d09e8ee323ebca2c44 2024-11-20T06:41:23,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cc4a6e38de4e4c07a7bead47823e4a60 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/cc4a6e38de4e4c07a7bead47823e4a60 2024-11-20T06:41:23,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad693bb5313540f6989861ec7d65eeb9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad693bb5313540f6989861ec7d65eeb9 2024-11-20T06:41:23,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c0e12c35ebbf4254a6ace621db99a217 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c0e12c35ebbf4254a6ace621db99a217 2024-11-20T06:41:23,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f0df072cbdaf494ba53929fdf0fcb767 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/f0df072cbdaf494ba53929fdf0fcb767 2024-11-20T06:41:23,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad4a3f04852541879ebdfd122bb54970 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ad4a3f04852541879ebdfd122bb54970 2024-11-20T06:41:23,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8f0573648e1340afbd1a8ef1781be35f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8f0573648e1340afbd1a8ef1781be35f 2024-11-20T06:41:23,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e2a095963dd144a2a1ff9f0ae221d53c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e2a095963dd144a2a1ff9f0ae221d53c 2024-11-20T06:41:23,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/69b6ce941c5b425387d769841c340336 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/69b6ce941c5b425387d769841c340336 2024-11-20T06:41:23,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e482706c11ab479880efbd02db43b979 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/e482706c11ab479880efbd02db43b979 2024-11-20T06:41:23,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ca0ae88bfa3644dcb18a17ab3cabdc61 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ca0ae88bfa3644dcb18a17ab3cabdc61 2024-11-20T06:41:23,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/6b103e83f3964fbea64286189147f14b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/6b103e83f3964fbea64286189147f14b 2024-11-20T06:41:23,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/82e8a739d3eb41bf819426c0a322c6fb to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/82e8a739d3eb41bf819426c0a322c6fb 2024-11-20T06:41:23,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/18406014a0614a2aae5cba694fd39e99 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/18406014a0614a2aae5cba694fd39e99 2024-11-20T06:41:23,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95fd8fddda6748259e284fb274d34ddf to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/95fd8fddda6748259e284fb274d34ddf 2024-11-20T06:41:23,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8939d1239cf14ff0a3356ac434414d37 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/8939d1239cf14ff0a3356ac434414d37 2024-11-20T06:41:23,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/a52005c66da043d1bc5481c6c9a3ea04 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/a52005c66da043d1bc5481c6c9a3ea04 2024-11-20T06:41:23,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c911a07d56dc41a7aed466b0b617882e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/c911a07d56dc41a7aed466b0b617882e 2024-11-20T06:41:23,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/7354c0e3eadd4a2985b64fc6395d813e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/7354c0e3eadd4a2985b64fc6395d813e 2024-11-20T06:41:23,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ef89eac5ffd349b28023ca52263e9f17 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/ef89eac5ffd349b28023ca52263e9f17 2024-11-20T06:41:23,565 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/recovered.edits/522.seqid, newMaxSeqId=522, maxSeqId=1 2024-11-20T06:41:23,566 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. 2024-11-20T06:41:23,566 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 792921c2784cd0f6a5ba47b7f4462f5c: Waiting for close lock at 1732084882191Running coprocessor pre-close hooks at 1732084882191Disabling compacts and flushes for region at 1732084882191Disabling writes for close at 1732084882191Obtaining lock to block concurrent updates at 1732084882191Preparing flush snapshotting stores in 792921c2784cd0f6a5ba47b7f4462f5c at 1732084882191Finished memstore snapshotting TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732084882191Flushing stores of TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c. at 1732084882192 (+1 ms)Flushing 792921c2784cd0f6a5ba47b7f4462f5c/A: creating writer at 1732084882192Flushing 792921c2784cd0f6a5ba47b7f4462f5c/A: appending metadata at 1732084882194 (+2 ms)Flushing 792921c2784cd0f6a5ba47b7f4462f5c/A: closing flushed file at 1732084882194Flushing 792921c2784cd0f6a5ba47b7f4462f5c/B: creating writer at 1732084882603 (+409 ms)Flushing 792921c2784cd0f6a5ba47b7f4462f5c/B: appending metadata at 1732084882605 (+2 ms)Flushing 792921c2784cd0f6a5ba47b7f4462f5c/B: closing flushed file at 1732084882605Flushing 792921c2784cd0f6a5ba47b7f4462f5c/C: creating writer at 1732084883015 (+410 ms)Flushing 792921c2784cd0f6a5ba47b7f4462f5c/C: appending metadata at 1732084883018 (+3 ms)Flushing 792921c2784cd0f6a5ba47b7f4462f5c/C: closing flushed file at 1732084883018Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d9b5866: reopening flushed file at 1732084883426 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21a1584d: reopening flushed file at 1732084883431 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e86c5b1: reopening flushed file at 1732084883435 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 792921c2784cd0f6a5ba47b7f4462f5c in 1250ms, sequenceid=519, compaction requested=true at 1732084883441 (+6 ms)Writing region close event to WAL at 1732084883562 (+121 ms)Running coprocessor post-close hooks at 1732084883566 (+4 ms)Closed at 1732084883566 2024-11-20T06:41:23,568 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:23,569 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=792921c2784cd0f6a5ba47b7f4462f5c, regionState=CLOSED 2024-11-20T06:41:23,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:41:23,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-20T06:41:23,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 792921c2784cd0f6a5ba47b7f4462f5c, server=61c2066a8a0f,33847,1732084794617 in 1.5330 sec 2024-11-20T06:41:23,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-20T06:41:23,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=792921c2784cd0f6a5ba47b7f4462f5c, UNASSIGN in 1.5370 sec 2024-11-20T06:41:23,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-20T06:41:23,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5420 sec 2024-11-20T06:41:23,577 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084883577"}]},"ts":"1732084883577"} 2024-11-20T06:41:23,578 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T06:41:23,578 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T06:41:23,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5510 sec 2024-11-20T06:41:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T06:41:24,174 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:24,175 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T06:41:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:24,176 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T06:41:24,177 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:24,179 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:24,181 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/recovered.edits] 2024-11-20T06:41:24,184 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/13e0765a1c5647b6a968f30b59befb61 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/13e0765a1c5647b6a968f30b59befb61 2024-11-20T06:41:24,185 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/7645bef31568449f8457a68aaa6fce10 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/7645bef31568449f8457a68aaa6fce10 2024-11-20T06:41:24,186 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/af860f7b5458460c86eaf7e15267d560 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/A/af860f7b5458460c86eaf7e15267d560 2024-11-20T06:41:24,188 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/336300e3226c4613b223e5f29c170ea5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/336300e3226c4613b223e5f29c170ea5 2024-11-20T06:41:24,189 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/457bf5b9d8d34855bfe3090dee9a162b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/457bf5b9d8d34855bfe3090dee9a162b 2024-11-20T06:41:24,190 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8ee81ffdc4944ead8be5d47266fce50b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/B/8ee81ffdc4944ead8be5d47266fce50b 2024-11-20T06:41:24,192 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/0da6ccce3ac84a98a15a44ccd4068e24 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/0da6ccce3ac84a98a15a44ccd4068e24 2024-11-20T06:41:24,194 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/47b0f6781b274d0281b7cf0eec51d1b6 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/47b0f6781b274d0281b7cf0eec51d1b6 2024-11-20T06:41:24,195 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/689b3ff144754debb50bab45db153463 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/C/689b3ff144754debb50bab45db153463 2024-11-20T06:41:24,197 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/recovered.edits/522.seqid to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c/recovered.edits/522.seqid 2024-11-20T06:41:24,198 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/792921c2784cd0f6a5ba47b7f4462f5c 2024-11-20T06:41:24,198 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T06:41:24,200 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:24,204 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T06:41:24,206 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T06:41:24,207 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:24,207 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T06:41:24,207 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732084884207"}]},"ts":"9223372036854775807"} 2024-11-20T06:41:24,209 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T06:41:24,209 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 792921c2784cd0f6a5ba47b7f4462f5c, NAME => 'TestAcidGuarantees,,1732084853867.792921c2784cd0f6a5ba47b7f4462f5c.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T06:41:24,209 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T06:41:24,210 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732084884209"}]},"ts":"9223372036854775807"} 2024-11-20T06:41:24,211 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T06:41:24,212 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:24,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-11-20T06:41:24,221 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T06:41:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T06:41:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T06:41:24,288 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:24,295 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 240), OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=446 (was 467), ProcessCount=11 (was 11), AvailableMemoryMB=4601 (was 4040) - AvailableMemoryMB LEAK? - 2024-11-20T06:41:24,304 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=446, ProcessCount=11, AvailableMemoryMB=4601 2024-11-20T06:41:24,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T06:41:24,306 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:41:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:24,308 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T06:41:24,308 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:24,309 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-20T06:41:24,309 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T06:41:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T06:41:24,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742167_1343 (size=959) 2024-11-20T06:41:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T06:41:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T06:41:24,718 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5 2024-11-20T06:41:24,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742168_1344 (size=53) 2024-11-20T06:41:24,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:41:24,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 88c2857bcc22b20889f07648453bfe35, disabling compactions & flushes 2024-11-20T06:41:24,723 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:24,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:24,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. after waiting 0 ms 2024-11-20T06:41:24,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:24,723 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:24,723 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 88c2857bcc22b20889f07648453bfe35: Waiting for close lock at 1732084884723Disabling compacts and flushes for region at 1732084884723Disabling writes for close at 1732084884723Writing region close event to WAL at 1732084884723Closed at 1732084884723 2024-11-20T06:41:24,724 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T06:41:24,725 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732084884724"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732084884724"}]},"ts":"1732084884724"} 2024-11-20T06:41:24,726 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T06:41:24,727 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T06:41:24,727 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084884727"}]},"ts":"1732084884727"} 2024-11-20T06:41:24,729 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T06:41:24,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, ASSIGN}] 2024-11-20T06:41:24,730 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, ASSIGN 2024-11-20T06:41:24,731 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, ASSIGN; state=OFFLINE, location=61c2066a8a0f,33847,1732084794617; forceNewPlan=false, retain=false 2024-11-20T06:41:24,882 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=88c2857bcc22b20889f07648453bfe35, regionState=OPENING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:24,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, ASSIGN because future has completed 2024-11-20T06:41:24,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:41:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T06:41:25,038 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:25,038 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:41:25,038 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,038 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:41:25,038 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,038 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,040 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,041 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88c2857bcc22b20889f07648453bfe35 columnFamilyName A 2024-11-20T06:41:25,041 DEBUG [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:25,042 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(327): Store=88c2857bcc22b20889f07648453bfe35/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:41:25,042 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,043 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88c2857bcc22b20889f07648453bfe35 columnFamilyName B 2024-11-20T06:41:25,043 DEBUG [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:25,044 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(327): Store=88c2857bcc22b20889f07648453bfe35/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:41:25,044 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,045 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88c2857bcc22b20889f07648453bfe35 columnFamilyName C 2024-11-20T06:41:25,045 DEBUG [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:25,046 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(327): Store=88c2857bcc22b20889f07648453bfe35/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:41:25,046 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,047 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,047 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,049 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,049 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,049 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:41:25,050 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,053 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:41:25,053 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 88c2857bcc22b20889f07648453bfe35; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71062387, jitterRate=0.05891208350658417}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:41:25,053 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:25,054 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 88c2857bcc22b20889f07648453bfe35: Running coprocessor pre-open hook at 1732084885038Writing region info on filesystem at 1732084885038Initializing all the Stores at 1732084885039 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084885039Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084885040 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084885040Cleaning up temporary data from old regions at 1732084885049 (+9 ms)Running coprocessor post-open hooks at 1732084885053 (+4 ms)Region opened successfully at 1732084885054 (+1 ms) 2024-11-20T06:41:25,055 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., pid=93, masterSystemTime=1732084885035 2024-11-20T06:41:25,057 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:25,057 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:25,057 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=88c2857bcc22b20889f07648453bfe35, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:25,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:41:25,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-20T06:41:25,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 in 175 msec 2024-11-20T06:41:25,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-20T06:41:25,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, ASSIGN in 333 msec 2024-11-20T06:41:25,065 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T06:41:25,065 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084885065"}]},"ts":"1732084885065"} 2024-11-20T06:41:25,067 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T06:41:25,067 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T06:41:25,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 761 msec 2024-11-20T06:41:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T06:41:25,444 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:25,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48dcb24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:25,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:25,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:25,446 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f488d99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:25,448 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49025b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:25,449 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:25,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:25,451 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T06:41:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,33659,1732084793752 2024-11-20T06:41:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@62009ef7 2024-11-20T06:41:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T06:41:25,453 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T06:41:25,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T06:41:25,454 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:41:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:25,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742169_1345 (size=995) 2024-11-20T06:41:25,867 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T06:41:25,867 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T06:41:25,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T06:41:25,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, REOPEN/MOVE}] 2024-11-20T06:41:25,873 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, REOPEN/MOVE 2024-11-20T06:41:25,874 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=88c2857bcc22b20889f07648453bfe35, regionState=CLOSING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:25,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, REOPEN/MOVE because future has completed 2024-11-20T06:41:25,875 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T06:41:25,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:41:26,028 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,028 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T06:41:26,028 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 88c2857bcc22b20889f07648453bfe35, disabling compactions & flushes 2024-11-20T06:41:26,028 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,028 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,028 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. after waiting 0 ms 2024-11-20T06:41:26,028 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,032 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T06:41:26,033 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,033 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 88c2857bcc22b20889f07648453bfe35: Waiting for close lock at 1732084886028Running coprocessor pre-close hooks at 1732084886028Disabling compacts and flushes for region at 1732084886028Disabling writes for close at 1732084886028Writing region close event to WAL at 1732084886029 (+1 ms)Running coprocessor post-close hooks at 1732084886032 (+3 ms)Closed at 1732084886033 (+1 ms) 2024-11-20T06:41:26,033 WARN [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 88c2857bcc22b20889f07648453bfe35 to self. 2024-11-20T06:41:26,034 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,035 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=88c2857bcc22b20889f07648453bfe35, regionState=CLOSED 2024-11-20T06:41:26,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:41:26,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-20T06:41:26,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 in 162 msec 2024-11-20T06:41:26,039 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, REOPEN/MOVE; state=CLOSED, location=61c2066a8a0f,33847,1732084794617; forceNewPlan=false, retain=true 2024-11-20T06:41:26,190 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=88c2857bcc22b20889f07648453bfe35, regionState=OPENING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, REOPEN/MOVE because future has completed 2024-11-20T06:41:26,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:41:26,349 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,349 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:41:26,349 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,349 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:41:26,349 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,349 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,350 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,351 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88c2857bcc22b20889f07648453bfe35 columnFamilyName A 2024-11-20T06:41:26,352 DEBUG [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:26,352 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(327): Store=88c2857bcc22b20889f07648453bfe35/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:41:26,353 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,353 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88c2857bcc22b20889f07648453bfe35 columnFamilyName B 2024-11-20T06:41:26,354 DEBUG [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:26,354 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(327): Store=88c2857bcc22b20889f07648453bfe35/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:41:26,354 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,355 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88c2857bcc22b20889f07648453bfe35 columnFamilyName C 2024-11-20T06:41:26,355 DEBUG [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:26,355 INFO [StoreOpener-88c2857bcc22b20889f07648453bfe35-1 {}] regionserver.HStore(327): Store=88c2857bcc22b20889f07648453bfe35/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:41:26,355 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,356 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,357 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,357 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,358 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,358 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:41:26,359 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,360 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 88c2857bcc22b20889f07648453bfe35; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64907470, jitterRate=-0.03280332684516907}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:41:26,360 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,361 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 88c2857bcc22b20889f07648453bfe35: Running coprocessor pre-open hook at 1732084886349Writing region info on filesystem at 1732084886349Initializing all the Stores at 1732084886350 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084886350Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084886350Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084886350Cleaning up temporary data from old regions at 1732084886358 (+8 ms)Running coprocessor post-open hooks at 1732084886360 (+2 ms)Region opened successfully at 1732084886360 2024-11-20T06:41:26,361 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., pid=98, masterSystemTime=1732084886346 2024-11-20T06:41:26,363 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,363 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,364 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=88c2857bcc22b20889f07648453bfe35, regionState=OPEN, openSeqNum=5, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:41:26,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-20T06:41:26,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 in 174 msec 2024-11-20T06:41:26,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-20T06:41:26,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, REOPEN/MOVE in 496 msec 2024-11-20T06:41:26,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-20T06:41:26,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 502 msec 2024-11-20T06:41:26,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 918 msec 2024-11-20T06:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-20T06:41:26,376 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:26,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5232d48c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:26,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:26,377 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25d6d4d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:26,378 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:26,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2010baed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b67a6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:26,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:26,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279cd7ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:26,381 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:26,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@543af064, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12688331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,382 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:26,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:26,383 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16c8e1a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:26,384 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dec084d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1456e4e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:26,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:26,387 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3792a43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:26,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:26,388 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b1f0cc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32da4b0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:26,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:26,390 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:26,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:26,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f0ee91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:26,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e71f9c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15099ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:26,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:26,393 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38893eb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:26,395 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:26,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d6dd65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbfebb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:26,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:26,396 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:26,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:26,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:26,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34396bd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:26,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:26,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:26,397 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:26,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@568457cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bdde2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,398 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:26,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:26,399 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1f9cde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:26,401 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:26,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bfaafa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7608485b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:26,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:26,402 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:26,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625e3708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:26,404 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dbf14c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,404 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf0e288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:26,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:26,405 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@105beea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:26,407 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:26,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2384322c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:26,409 DEBUG [Thread-1570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:26,409 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:26,409 DEBUG [Thread-1575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:26,409 DEBUG [Thread-1581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:26,410 DEBUG [Thread-1579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:26,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:26,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:26,411 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:26,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:26,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:26,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:26,412 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:26,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:41:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:26,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:41:26,416 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-20T06:41:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T06:41:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:41:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:41:26,423 DEBUG [Thread-1577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:41:26,423 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:26,424 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:26,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,428 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:26,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:41:26,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:41:26,436 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:26,439 DEBUG [Thread-1583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:26,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:41:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:26,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:26,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:26,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:41:26,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204ebf0ae346f94e098acf1b00cbed1c64_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084886419/Put/seqid=0 2024-11-20T06:41:26,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742170_1346 (size=17034) 2024-11-20T06:41:26,458 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:26,462 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204ebf0ae346f94e098acf1b00cbed1c64_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204ebf0ae346f94e098acf1b00cbed1c64_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:26,463 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/943e657785c7457eb4c2fc0728dfdd55, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:26,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/943e657785c7457eb4c2fc0728dfdd55 is 175, key is test_row_0/A:col10/1732084886419/Put/seqid=0 2024-11-20T06:41:26,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742171_1347 (size=48139) 2024-11-20T06:41:26,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084946473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084946474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084946477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084946475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084946484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T06:41:26,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T06:41:26,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:26,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:26,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:26,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084946594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084946594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084946594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084946595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084946595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T06:41:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:26,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T06:41:26,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084946803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084946803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084946803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084946803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084946804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:26,880 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/943e657785c7457eb4c2fc0728dfdd55 2024-11-20T06:41:26,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T06:41:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:26,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:26,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/03287615e237473a8888665112ffc747 is 50, key is test_row_0/B:col10/1732084886419/Put/seqid=0 2024-11-20T06:41:26,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742172_1348 (size=12001) 2024-11-20T06:41:27,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T06:41:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T06:41:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084947114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084947114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084947114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084947114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084947114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:27,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T06:41:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/03287615e237473a8888665112ffc747 2024-11-20T06:41:27,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/cf854552520d43c8979bc5800cfbb1dd is 50, key is test_row_0/C:col10/1732084886419/Put/seqid=0 2024-11-20T06:41:27,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742173_1349 (size=12001) 2024-11-20T06:41:27,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T06:41:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T06:41:27,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:27,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T06:41:27,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084947623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084947623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084947623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:27,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084947623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:27,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084947624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:27,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T06:41:27,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:27,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:27,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/cf854552520d43c8979bc5800cfbb1dd 2024-11-20T06:41:27,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/943e657785c7457eb4c2fc0728dfdd55 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/943e657785c7457eb4c2fc0728dfdd55 2024-11-20T06:41:27,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/943e657785c7457eb4c2fc0728dfdd55, entries=250, sequenceid=15, filesize=47.0 K 2024-11-20T06:41:27,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/03287615e237473a8888665112ffc747 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/03287615e237473a8888665112ffc747 2024-11-20T06:41:27,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/03287615e237473a8888665112ffc747, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T06:41:27,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/cf854552520d43c8979bc5800cfbb1dd as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/cf854552520d43c8979bc5800cfbb1dd 2024-11-20T06:41:27,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/cf854552520d43c8979bc5800cfbb1dd, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T06:41:27,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 88c2857bcc22b20889f07648453bfe35 in 1327ms, sequenceid=15, compaction requested=false 2024-11-20T06:41:27,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:27,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T06:41:27,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:27,803 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T06:41:27,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f88399ab5e4b49c189ad1e93743e8296_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084886474/Put/seqid=0 2024-11-20T06:41:27,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742174_1350 (size=12154) 2024-11-20T06:41:27,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:27,837 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f88399ab5e4b49c189ad1e93743e8296_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f88399ab5e4b49c189ad1e93743e8296_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:27,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6e981b2c9c1449098ba6ffadd97d9aef, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:27,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6e981b2c9c1449098ba6ffadd97d9aef is 175, key is test_row_0/A:col10/1732084886474/Put/seqid=0 2024-11-20T06:41:27,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742175_1351 (size=30955) 2024-11-20T06:41:28,245 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6e981b2c9c1449098ba6ffadd97d9aef 2024-11-20T06:41:28,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/127d7be71aac427eba3d20a810255cc7 is 50, key is test_row_0/B:col10/1732084886474/Put/seqid=0 2024-11-20T06:41:28,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742176_1352 (size=12001) 2024-11-20T06:41:28,262 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/127d7be71aac427eba3d20a810255cc7 2024-11-20T06:41:28,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/daec66f8766e4ec78ebf210cad3575e7 is 50, key is test_row_0/C:col10/1732084886474/Put/seqid=0 2024-11-20T06:41:28,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742177_1353 (size=12001) 2024-11-20T06:41:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T06:41:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:28,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084948653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084948653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084948661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084948661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084948661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,672 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/daec66f8766e4ec78ebf210cad3575e7 2024-11-20T06:41:28,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6e981b2c9c1449098ba6ffadd97d9aef as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6e981b2c9c1449098ba6ffadd97d9aef 2024-11-20T06:41:28,679 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6e981b2c9c1449098ba6ffadd97d9aef, entries=150, sequenceid=40, filesize=30.2 K 2024-11-20T06:41:28,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/127d7be71aac427eba3d20a810255cc7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/127d7be71aac427eba3d20a810255cc7 2024-11-20T06:41:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,682 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/127d7be71aac427eba3d20a810255cc7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T06:41:28,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/daec66f8766e4ec78ebf210cad3575e7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/daec66f8766e4ec78ebf210cad3575e7 2024-11-20T06:41:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,687 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/daec66f8766e4ec78ebf210cad3575e7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T06:41:28,688 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 88c2857bcc22b20889f07648453bfe35 in 884ms, sequenceid=40, compaction requested=false 2024-11-20T06:41:28,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:28,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:28,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-20T06:41:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-20T06:41:28,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-20T06:41:28,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2650 sec 2024-11-20T06:41:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.2750 sec 2024-11-20T06:41:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:28,794 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:28,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T06:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200a7fbbe1289f407996a03371e970513b_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084888794/Put/seqid=0 2024-11-20T06:41:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742178_1354 (size=14594) 2024-11-20T06:41:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,812 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,815 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200a7fbbe1289f407996a03371e970513b_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200a7fbbe1289f407996a03371e970513b_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,816 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6c5380bf170b495cb0d2744cd111e09c, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6c5380bf170b495cb0d2744cd111e09c is 175, key is test_row_0/A:col10/1732084888794/Put/seqid=0 2024-11-20T06:41:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742179_1355 (size=39549) 2024-11-20T06:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:28,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084948846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084948845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084948851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084948852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084948856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084948964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084948964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084948964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084948973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084948973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:28,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,095 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T06:41:29,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084949174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084949184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084949184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084949185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084949185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,221 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6c5380bf170b495cb0d2744cd111e09c 2024-11-20T06:41:29,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/b1cf06610fc5445abe37b3d86e1b116b is 50, key is test_row_0/B:col10/1732084888794/Put/seqid=0 2024-11-20T06:41:29,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742180_1356 (size=12001) 2024-11-20T06:41:29,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084949494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:29,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084949503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084949503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084949503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084949504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:29,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/b1cf06610fc5445abe37b3d86e1b116b 2024-11-20T06:41:29,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/11fc8da624f44b998c78c94312ef9800 is 50, key is test_row_0/C:col10/1732084888794/Put/seqid=0 2024-11-20T06:41:29,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742181_1357 (size=12001) 2024-11-20T06:41:30,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084950003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:30,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084950014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:30,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084950014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:30,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084950014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:30,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084950014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:30,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/11fc8da624f44b998c78c94312ef9800 2024-11-20T06:41:30,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6c5380bf170b495cb0d2744cd111e09c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6c5380bf170b495cb0d2744cd111e09c 2024-11-20T06:41:30,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6c5380bf170b495cb0d2744cd111e09c, entries=200, sequenceid=55, filesize=38.6 K 2024-11-20T06:41:30,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/b1cf06610fc5445abe37b3d86e1b116b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/b1cf06610fc5445abe37b3d86e1b116b 2024-11-20T06:41:30,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/b1cf06610fc5445abe37b3d86e1b116b, entries=150, sequenceid=55, filesize=11.7 K 2024-11-20T06:41:30,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/11fc8da624f44b998c78c94312ef9800 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/11fc8da624f44b998c78c94312ef9800 2024-11-20T06:41:30,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/11fc8da624f44b998c78c94312ef9800, entries=150, sequenceid=55, filesize=11.7 K 2024-11-20T06:41:30,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 88c2857bcc22b20889f07648453bfe35 in 1263ms, sequenceid=55, compaction requested=true 2024-11-20T06:41:30,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:30,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:30,058 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:30,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:30,058 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:30,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:30,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:30,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:30,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:30,058 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:30,058 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:30,058 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/A is initiating minor compaction (all files) 2024-11-20T06:41:30,058 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/B is initiating minor compaction (all files) 2024-11-20T06:41:30,059 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/A in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:30,059 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/B in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:30,059 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/943e657785c7457eb4c2fc0728dfdd55, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6e981b2c9c1449098ba6ffadd97d9aef, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6c5380bf170b495cb0d2744cd111e09c] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=115.9 K 2024-11-20T06:41:30,059 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/03287615e237473a8888665112ffc747, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/127d7be71aac427eba3d20a810255cc7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/b1cf06610fc5445abe37b3d86e1b116b] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=35.2 K 2024-11-20T06:41:30,059 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:30,059 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/943e657785c7457eb4c2fc0728dfdd55, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6e981b2c9c1449098ba6ffadd97d9aef, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6c5380bf170b495cb0d2744cd111e09c] 2024-11-20T06:41:30,059 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 03287615e237473a8888665112ffc747, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732084886419 2024-11-20T06:41:30,059 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 943e657785c7457eb4c2fc0728dfdd55, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732084886419 2024-11-20T06:41:30,059 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e981b2c9c1449098ba6ffadd97d9aef, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732084886473 2024-11-20T06:41:30,059 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 127d7be71aac427eba3d20a810255cc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732084886473 2024-11-20T06:41:30,059 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c5380bf170b495cb0d2744cd111e09c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732084888659 2024-11-20T06:41:30,059 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b1cf06610fc5445abe37b3d86e1b116b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732084888660 2024-11-20T06:41:30,065 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:30,066 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/fe3abab9d8924cb0ae5ebd2c2f720532 is 50, key is test_row_0/B:col10/1732084888794/Put/seqid=0 2024-11-20T06:41:30,067 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:30,072 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120b31e8839aceb423a93dec3a5562bd72e_88c2857bcc22b20889f07648453bfe35 store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:30,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742182_1358 (size=12104) 2024-11-20T06:41:30,074 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120b31e8839aceb423a93dec3a5562bd72e_88c2857bcc22b20889f07648453bfe35, store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:30,074 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b31e8839aceb423a93dec3a5562bd72e_88c2857bcc22b20889f07648453bfe35 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:30,077 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/fe3abab9d8924cb0ae5ebd2c2f720532 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/fe3abab9d8924cb0ae5ebd2c2f720532 2024-11-20T06:41:30,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742183_1359 (size=4469) 2024-11-20T06:41:30,082 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/B of 88c2857bcc22b20889f07648453bfe35 into fe3abab9d8924cb0ae5ebd2c2f720532(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:30,082 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:30,082 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/B, priority=13, startTime=1732084890058; duration=0sec 2024-11-20T06:41:30,082 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:30,082 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:B 2024-11-20T06:41:30,082 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:30,085 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:30,085 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/C is initiating minor compaction (all files) 2024-11-20T06:41:30,085 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/C in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:30,086 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/cf854552520d43c8979bc5800cfbb1dd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/daec66f8766e4ec78ebf210cad3575e7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/11fc8da624f44b998c78c94312ef9800] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=35.2 K 2024-11-20T06:41:30,086 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting cf854552520d43c8979bc5800cfbb1dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732084886419 2024-11-20T06:41:30,086 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting daec66f8766e4ec78ebf210cad3575e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732084886473 2024-11-20T06:41:30,086 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 11fc8da624f44b998c78c94312ef9800, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732084888660 2024-11-20T06:41:30,092 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:30,093 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/8ea59497a34f459dbaabc8f5d615e7d0 is 50, key is test_row_0/C:col10/1732084888794/Put/seqid=0 2024-11-20T06:41:30,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742184_1360 (size=12104) 2024-11-20T06:41:30,481 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#A#compaction#313 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:30,481 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/4ede75d69b9d45089eb1db5d07a69100 is 175, key is test_row_0/A:col10/1732084888794/Put/seqid=0 2024-11-20T06:41:30,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742185_1361 (size=31058) 2024-11-20T06:41:30,507 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/8ea59497a34f459dbaabc8f5d615e7d0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/8ea59497a34f459dbaabc8f5d615e7d0 2024-11-20T06:41:30,511 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/C of 88c2857bcc22b20889f07648453bfe35 into 8ea59497a34f459dbaabc8f5d615e7d0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:30,512 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:30,512 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/C, priority=13, startTime=1732084890058; duration=0sec 2024-11-20T06:41:30,512 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:30,512 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:C 2024-11-20T06:41:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T06:41:30,574 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:30,575 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-20T06:41:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T06:41:30,576 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:30,577 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:30,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T06:41:30,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T06:41:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:30,729 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T06:41:30,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120baef0ab2393e4e9fa2697ae5f688921c_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084888843/Put/seqid=0 2024-11-20T06:41:30,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742186_1362 (size=12154) 2024-11-20T06:41:30,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:30,790 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120baef0ab2393e4e9fa2697ae5f688921c_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120baef0ab2393e4e9fa2697ae5f688921c_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:30,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/2894173cc86644959f5f8500e2445345, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:30,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/2894173cc86644959f5f8500e2445345 is 175, key is test_row_0/A:col10/1732084888843/Put/seqid=0 2024-11-20T06:41:30,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742187_1363 (size=30955) 2024-11-20T06:41:30,795 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/2894173cc86644959f5f8500e2445345 2024-11-20T06:41:30,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/a9863e2afbaa488fae1c7562a1562bae is 50, key is test_row_0/B:col10/1732084888843/Put/seqid=0 2024-11-20T06:41:30,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742188_1364 (size=12001) 2024-11-20T06:41:30,817 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/a9863e2afbaa488fae1c7562a1562bae 2024-11-20T06:41:30,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/e928a5cd27e842709d2733502caaff79 is 50, key is test_row_0/C:col10/1732084888843/Put/seqid=0 2024-11-20T06:41:30,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742189_1365 (size=12001) 2024-11-20T06:41:30,889 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/4ede75d69b9d45089eb1db5d07a69100 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4ede75d69b9d45089eb1db5d07a69100 2024-11-20T06:41:30,893 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/A of 88c2857bcc22b20889f07648453bfe35 into 4ede75d69b9d45089eb1db5d07a69100(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:30,893 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:30,893 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/A, priority=13, startTime=1732084890058; duration=0sec 2024-11-20T06:41:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T06:41:30,893 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:30,893 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:A 2024-11-20T06:41:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:31,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:31,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084951047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084951052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084951053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084951055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084951056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084951163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084951163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084951174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084951174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084951174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T06:41:31,228 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/e928a5cd27e842709d2733502caaff79 2024-11-20T06:41:31,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/2894173cc86644959f5f8500e2445345 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/2894173cc86644959f5f8500e2445345 2024-11-20T06:41:31,235 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/2894173cc86644959f5f8500e2445345, entries=150, sequenceid=78, filesize=30.2 K 2024-11-20T06:41:31,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/a9863e2afbaa488fae1c7562a1562bae as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a9863e2afbaa488fae1c7562a1562bae 2024-11-20T06:41:31,238 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a9863e2afbaa488fae1c7562a1562bae, entries=150, sequenceid=78, filesize=11.7 K 2024-11-20T06:41:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/e928a5cd27e842709d2733502caaff79 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/e928a5cd27e842709d2733502caaff79 2024-11-20T06:41:31,242 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/e928a5cd27e842709d2733502caaff79, entries=150, sequenceid=78, filesize=11.7 K 2024-11-20T06:41:31,243 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 88c2857bcc22b20889f07648453bfe35 in 514ms, sequenceid=78, compaction requested=false 2024-11-20T06:41:31,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:31,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:31,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-20T06:41:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-20T06:41:31,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-20T06:41:31,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 668 msec 2024-11-20T06:41:31,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 671 msec 2024-11-20T06:41:31,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:31,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:31,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T06:41:31,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203ce56a72f5134d458c8461f863b950a3_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084891051/Put/seqid=0 2024-11-20T06:41:31,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742190_1366 (size=14594) 2024-11-20T06:41:31,391 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:31,394 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203ce56a72f5134d458c8461f863b950a3_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203ce56a72f5134d458c8461f863b950a3_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:31,395 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/69d6556c6b7d4cb987009b7f34416861, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:31,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/69d6556c6b7d4cb987009b7f34416861 is 175, key is test_row_0/A:col10/1732084891051/Put/seqid=0 2024-11-20T06:41:31,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742191_1367 (size=39549) 2024-11-20T06:41:31,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084951409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084951409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084951413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084951414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084951415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084951523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084951523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084951534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084951534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084951534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T06:41:31,714 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:31,715 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-20T06:41:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T06:41:31,716 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:31,717 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:31,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:31,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084951734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084951734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084951744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084951754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084951754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:31,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:31,798 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/69d6556c6b7d4cb987009b7f34416861 2024-11-20T06:41:31,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/adee63115aff4100b345177a343d9b1b is 50, key is test_row_0/B:col10/1732084891051/Put/seqid=0 2024-11-20T06:41:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T06:41:31,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742192_1368 (size=12001) 2024-11-20T06:41:31,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T06:41:31,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:31,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:31,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:31,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:31,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T06:41:32,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:32,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T06:41:32,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084952054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:32,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084952054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:32,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:32,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084952064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084952073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084952073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:32,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T06:41:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/adee63115aff4100b345177a343d9b1b 2024-11-20T06:41:32,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/5296e1514a954e859fbee9b32d732583 is 50, key is test_row_0/C:col10/1732084891051/Put/seqid=0 2024-11-20T06:41:32,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742193_1369 (size=12001) 2024-11-20T06:41:32,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T06:41:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:32,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T06:41:32,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T06:41:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084952564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:32,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084952564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:32,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084952583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:32,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084952583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084952584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:32,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T06:41:32,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:32,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:32,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/5296e1514a954e859fbee9b32d732583 2024-11-20T06:41:32,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/69d6556c6b7d4cb987009b7f34416861 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/69d6556c6b7d4cb987009b7f34416861 2024-11-20T06:41:32,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/69d6556c6b7d4cb987009b7f34416861, entries=200, sequenceid=95, filesize=38.6 K 2024-11-20T06:41:32,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/adee63115aff4100b345177a343d9b1b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/adee63115aff4100b345177a343d9b1b 2024-11-20T06:41:32,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/adee63115aff4100b345177a343d9b1b, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T06:41:32,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/5296e1514a954e859fbee9b32d732583 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5296e1514a954e859fbee9b32d732583 2024-11-20T06:41:32,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5296e1514a954e859fbee9b32d732583, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T06:41:32,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 88c2857bcc22b20889f07648453bfe35 in 1273ms, sequenceid=95, compaction requested=true 2024-11-20T06:41:32,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:32,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:32,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:32,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:32,652 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:32,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:32,652 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:32,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:32,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:32,653 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:32,653 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:32,653 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/A is initiating minor compaction (all files) 2024-11-20T06:41:32,653 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/B is initiating minor compaction (all files) 2024-11-20T06:41:32,653 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/A in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,653 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/B in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,653 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/fe3abab9d8924cb0ae5ebd2c2f720532, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a9863e2afbaa488fae1c7562a1562bae, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/adee63115aff4100b345177a343d9b1b] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=35.3 K 2024-11-20T06:41:32,653 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4ede75d69b9d45089eb1db5d07a69100, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/2894173cc86644959f5f8500e2445345, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/69d6556c6b7d4cb987009b7f34416861] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=99.2 K 2024-11-20T06:41:32,653 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,653 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4ede75d69b9d45089eb1db5d07a69100, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/2894173cc86644959f5f8500e2445345, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/69d6556c6b7d4cb987009b7f34416861] 2024-11-20T06:41:32,654 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting fe3abab9d8924cb0ae5ebd2c2f720532, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732084888660 2024-11-20T06:41:32,654 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ede75d69b9d45089eb1db5d07a69100, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732084888660 2024-11-20T06:41:32,654 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a9863e2afbaa488fae1c7562a1562bae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732084888843 2024-11-20T06:41:32,654 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2894173cc86644959f5f8500e2445345, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732084888843 2024-11-20T06:41:32,654 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting adee63115aff4100b345177a343d9b1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732084891051 2024-11-20T06:41:32,654 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69d6556c6b7d4cb987009b7f34416861, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732084891051 2024-11-20T06:41:32,661 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:32,662 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:32,662 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d9c578bb5bf74bb291b56395cd9a2aa4 is 50, key is test_row_0/B:col10/1732084891051/Put/seqid=0 2024-11-20T06:41:32,662 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411207707163f376a44a183d83ca916b8980a_88c2857bcc22b20889f07648453bfe35 store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:32,664 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411207707163f376a44a183d83ca916b8980a_88c2857bcc22b20889f07648453bfe35, store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:32,664 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207707163f376a44a183d83ca916b8980a_88c2857bcc22b20889f07648453bfe35 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:32,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742194_1370 (size=12207) 2024-11-20T06:41:32,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742195_1371 (size=4469) 2024-11-20T06:41:32,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T06:41:32,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:32,787 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:41:32,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112039a16a299ee54be6895a12a73655a3ba_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084891411/Put/seqid=0 2024-11-20T06:41:32,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742196_1372 (size=12154) 2024-11-20T06:41:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T06:41:33,075 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d9c578bb5bf74bb291b56395cd9a2aa4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d9c578bb5bf74bb291b56395cd9a2aa4 2024-11-20T06:41:33,079 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/B of 88c2857bcc22b20889f07648453bfe35 into d9c578bb5bf74bb291b56395cd9a2aa4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:33,079 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:33,079 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/B, priority=13, startTime=1732084892652; duration=0sec 2024-11-20T06:41:33,079 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:33,079 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:B 2024-11-20T06:41:33,079 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:33,080 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:33,080 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/C is initiating minor compaction (all files) 2024-11-20T06:41:33,080 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/C in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:33,080 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/8ea59497a34f459dbaabc8f5d615e7d0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/e928a5cd27e842709d2733502caaff79, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5296e1514a954e859fbee9b32d732583] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=35.3 K 2024-11-20T06:41:33,081 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea59497a34f459dbaabc8f5d615e7d0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732084888660 2024-11-20T06:41:33,081 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting e928a5cd27e842709d2733502caaff79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732084888843 2024-11-20T06:41:33,081 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5296e1514a954e859fbee9b32d732583, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732084891051 2024-11-20T06:41:33,087 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:33,087 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/5471994daf7e425186ed1b92d8582169 is 50, key is test_row_0/C:col10/1732084891051/Put/seqid=0 2024-11-20T06:41:33,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742197_1373 (size=12207) 2024-11-20T06:41:33,097 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#A#compaction#322 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:33,098 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/ca5345cb95e84d6f97e20cc1226a0af4 is 175, key is test_row_0/A:col10/1732084891051/Put/seqid=0 2024-11-20T06:41:33,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742198_1374 (size=31161) 2024-11-20T06:41:33,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:33,202 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112039a16a299ee54be6895a12a73655a3ba_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112039a16a299ee54be6895a12a73655a3ba_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:33,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/a1e945e085aa4f9395cc99c23141cd24, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:33,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/a1e945e085aa4f9395cc99c23141cd24 is 175, key is test_row_0/A:col10/1732084891411/Put/seqid=0 2024-11-20T06:41:33,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742199_1375 (size=30955) 2024-11-20T06:41:33,495 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/5471994daf7e425186ed1b92d8582169 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5471994daf7e425186ed1b92d8582169 2024-11-20T06:41:33,499 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/C of 88c2857bcc22b20889f07648453bfe35 into 5471994daf7e425186ed1b92d8582169(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:33,499 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:33,499 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/C, priority=13, startTime=1732084892652; duration=0sec 2024-11-20T06:41:33,499 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:33,499 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:C 2024-11-20T06:41:33,506 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/ca5345cb95e84d6f97e20cc1226a0af4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/ca5345cb95e84d6f97e20cc1226a0af4 2024-11-20T06:41:33,510 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/A of 88c2857bcc22b20889f07648453bfe35 into ca5345cb95e84d6f97e20cc1226a0af4(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:33,510 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:33,510 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/A, priority=13, startTime=1732084892652; duration=0sec 2024-11-20T06:41:33,511 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:33,511 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:A 2024-11-20T06:41:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:33,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:33,606 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/a1e945e085aa4f9395cc99c23141cd24 2024-11-20T06:41:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/ca7d2fcc6cfe4b7aa492d40a236a4fba is 50, key is test_row_0/B:col10/1732084891411/Put/seqid=0 2024-11-20T06:41:33,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742200_1376 (size=12001) 2024-11-20T06:41:33,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084953615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084953616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084953616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084953617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084953617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084953733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084953733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084953733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084953734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084953734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T06:41:33,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084953943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084953943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084953944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084953944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084953944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,018 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/ca7d2fcc6cfe4b7aa492d40a236a4fba 2024-11-20T06:41:34,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/9deec385e4c0401eaf3e97d38f35f49e is 50, key is test_row_0/C:col10/1732084891411/Put/seqid=0 2024-11-20T06:41:34,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742201_1377 (size=12001) 2024-11-20T06:41:34,221 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T06:41:34,221 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T06:41:34,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084954254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084954254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084954254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084954254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084954254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,436 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/9deec385e4c0401eaf3e97d38f35f49e 2024-11-20T06:41:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/a1e945e085aa4f9395cc99c23141cd24 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/a1e945e085aa4f9395cc99c23141cd24 2024-11-20T06:41:34,443 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/a1e945e085aa4f9395cc99c23141cd24, entries=150, sequenceid=115, filesize=30.2 K 2024-11-20T06:41:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/ca7d2fcc6cfe4b7aa492d40a236a4fba as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/ca7d2fcc6cfe4b7aa492d40a236a4fba 2024-11-20T06:41:34,447 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/ca7d2fcc6cfe4b7aa492d40a236a4fba, entries=150, sequenceid=115, filesize=11.7 K 2024-11-20T06:41:34,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/9deec385e4c0401eaf3e97d38f35f49e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/9deec385e4c0401eaf3e97d38f35f49e 2024-11-20T06:41:34,451 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/9deec385e4c0401eaf3e97d38f35f49e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-20T06:41:34,452 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 88c2857bcc22b20889f07648453bfe35 in 1665ms, sequenceid=115, compaction requested=false 2024-11-20T06:41:34,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:34,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:34,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-20T06:41:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-20T06:41:34,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-20T06:41:34,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7360 sec 2024-11-20T06:41:34,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.7400 sec 2024-11-20T06:41:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:34,769 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:34,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T06:41:34,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120adba2c65bc584cb2a6e8e893d3473f9c_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084893616/Put/seqid=0 2024-11-20T06:41:34,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742202_1378 (size=17234) 2024-11-20T06:41:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084954791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084954793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084954794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084954794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084954795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084954904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084954904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084954914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084954914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084954914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084955113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084955114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084955124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084955124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084955124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:35,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120adba2c65bc584cb2a6e8e893d3473f9c_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120adba2c65bc584cb2a6e8e893d3473f9c_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:35,189 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/3ece0a0d1fad4a34be13cd515e269708, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:35,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/3ece0a0d1fad4a34be13cd515e269708 is 175, key is test_row_0/A:col10/1732084893616/Put/seqid=0 2024-11-20T06:41:35,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742203_1379 (size=48339) 2024-11-20T06:41:35,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084955423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084955423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084955433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084955434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084955444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/3ece0a0d1fad4a34be13cd515e269708 2024-11-20T06:41:35,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d3514a5cfb664f348648c43fe97ba2fc is 50, key is test_row_0/B:col10/1732084893616/Put/seqid=0 2024-11-20T06:41:35,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742204_1380 (size=12151) 2024-11-20T06:41:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T06:41:35,874 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:35,875 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-20T06:41:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T06:41:35,916 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:35,916 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:35,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:35,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084955944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084955944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084955954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084955954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:35,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084955954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:35,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:35,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T06:41:36,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d3514a5cfb664f348648c43fe97ba2fc 2024-11-20T06:41:36,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/54a1dbd8fe8b4de7a7cde2689d49938e is 50, key is test_row_0/C:col10/1732084893616/Put/seqid=0 2024-11-20T06:41:36,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742205_1381 (size=12151) 2024-11-20T06:41:36,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T06:41:36,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:36,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:36,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T06:41:36,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T06:41:36,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:36,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:36,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:36,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T06:41:36,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:36,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:36,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:36,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/54a1dbd8fe8b4de7a7cde2689d49938e 2024-11-20T06:41:36,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/3ece0a0d1fad4a34be13cd515e269708 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/3ece0a0d1fad4a34be13cd515e269708 2024-11-20T06:41:36,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/3ece0a0d1fad4a34be13cd515e269708, entries=250, sequenceid=137, filesize=47.2 K 2024-11-20T06:41:36,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d3514a5cfb664f348648c43fe97ba2fc as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d3514a5cfb664f348648c43fe97ba2fc 2024-11-20T06:41:36,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d3514a5cfb664f348648c43fe97ba2fc, entries=150, sequenceid=137, filesize=11.9 K 2024-11-20T06:41:36,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/54a1dbd8fe8b4de7a7cde2689d49938e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/54a1dbd8fe8b4de7a7cde2689d49938e 2024-11-20T06:41:36,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/54a1dbd8fe8b4de7a7cde2689d49938e, entries=150, sequenceid=137, filesize=11.9 K 2024-11-20T06:41:36,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 88c2857bcc22b20889f07648453bfe35 in 1682ms, sequenceid=137, compaction requested=true 2024-11-20T06:41:36,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:36,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:36,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:36,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:36,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:36,451 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:36,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:36,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:36,451 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:36,452 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:36,452 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:36,452 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/B is initiating minor compaction (all files) 2024-11-20T06:41:36,452 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/A is initiating minor compaction (all files) 2024-11-20T06:41:36,452 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/B in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,452 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/A in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,452 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d9c578bb5bf74bb291b56395cd9a2aa4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/ca7d2fcc6cfe4b7aa492d40a236a4fba, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d3514a5cfb664f348648c43fe97ba2fc] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=35.5 K 2024-11-20T06:41:36,452 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/ca5345cb95e84d6f97e20cc1226a0af4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/a1e945e085aa4f9395cc99c23141cd24, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/3ece0a0d1fad4a34be13cd515e269708] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=107.9 K 2024-11-20T06:41:36,452 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,452 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/ca5345cb95e84d6f97e20cc1226a0af4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/a1e945e085aa4f9395cc99c23141cd24, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/3ece0a0d1fad4a34be13cd515e269708] 2024-11-20T06:41:36,452 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d9c578bb5bf74bb291b56395cd9a2aa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732084891051 2024-11-20T06:41:36,452 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca5345cb95e84d6f97e20cc1226a0af4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732084891051 2024-11-20T06:41:36,452 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ca7d2fcc6cfe4b7aa492d40a236a4fba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732084891407 2024-11-20T06:41:36,452 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1e945e085aa4f9395cc99c23141cd24, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732084891407 2024-11-20T06:41:36,453 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d3514a5cfb664f348648c43fe97ba2fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084893616 2024-11-20T06:41:36,453 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ece0a0d1fad4a34be13cd515e269708, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084893595 2024-11-20T06:41:36,458 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:36,459 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:36,459 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/91d1d434db01402d819c8e43013bc537 is 50, key is test_row_0/B:col10/1732084893616/Put/seqid=0 2024-11-20T06:41:36,463 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112095ebaa2eccf244179a4cf673e4722e8a_88c2857bcc22b20889f07648453bfe35 store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:36,465 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112095ebaa2eccf244179a4cf673e4722e8a_88c2857bcc22b20889f07648453bfe35, store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:36,465 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112095ebaa2eccf244179a4cf673e4722e8a_88c2857bcc22b20889f07648453bfe35 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:36,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742206_1382 (size=12459) 2024-11-20T06:41:36,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742207_1383 (size=4469) 2024-11-20T06:41:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T06:41:36,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T06:41:36,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,527 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T06:41:36,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f8ac24780c8b4f5c96add619d9b0af04_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084894793/Put/seqid=0 2024-11-20T06:41:36,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742208_1384 (size=12304) 2024-11-20T06:41:36,873 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/91d1d434db01402d819c8e43013bc537 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/91d1d434db01402d819c8e43013bc537 2024-11-20T06:41:36,877 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#A#compaction#331 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:36,877 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/B of 88c2857bcc22b20889f07648453bfe35 into 91d1d434db01402d819c8e43013bc537(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:36,877 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:36,877 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/B, priority=13, startTime=1732084896451; duration=0sec 2024-11-20T06:41:36,877 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:36,877 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/e25191d92f444957adef2c109ab5ce8b is 175, key is test_row_0/A:col10/1732084893616/Put/seqid=0 2024-11-20T06:41:36,877 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:B 2024-11-20T06:41:36,878 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:36,878 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:36,878 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/C is initiating minor compaction (all files) 2024-11-20T06:41:36,878 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/C in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:36,878 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5471994daf7e425186ed1b92d8582169, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/9deec385e4c0401eaf3e97d38f35f49e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/54a1dbd8fe8b4de7a7cde2689d49938e] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=35.5 K 2024-11-20T06:41:36,879 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5471994daf7e425186ed1b92d8582169, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732084891051 2024-11-20T06:41:36,879 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 9deec385e4c0401eaf3e97d38f35f49e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732084891407 2024-11-20T06:41:36,879 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 54a1dbd8fe8b4de7a7cde2689d49938e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084893616 2024-11-20T06:41:36,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742209_1385 (size=31413) 2024-11-20T06:41:36,886 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:36,886 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/e25191d92f444957adef2c109ab5ce8b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/e25191d92f444957adef2c109ab5ce8b 2024-11-20T06:41:36,887 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/d69ce46cde1e4316a3414f4709a6b521 is 50, key is test_row_0/C:col10/1732084893616/Put/seqid=0 2024-11-20T06:41:36,892 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/A of 88c2857bcc22b20889f07648453bfe35 into e25191d92f444957adef2c109ab5ce8b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:36,892 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:36,892 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/A, priority=13, startTime=1732084896451; duration=0sec 2024-11-20T06:41:36,892 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:36,892 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:A 2024-11-20T06:41:36,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742210_1386 (size=12459) 2024-11-20T06:41:36,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:36,940 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f8ac24780c8b4f5c96add619d9b0af04_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f8ac24780c8b4f5c96add619d9b0af04_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/accc5ca640484cb0b884f1cb49a2d357, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:36,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/accc5ca640484cb0b884f1cb49a2d357 is 175, key is test_row_0/A:col10/1732084894793/Put/seqid=0 2024-11-20T06:41:36,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742211_1387 (size=31105) 2024-11-20T06:41:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:36,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:37,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084956992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084956992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084956993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084956994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084956995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T06:41:37,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084957104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084957104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084957114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084957114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084957114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,297 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/d69ce46cde1e4316a3414f4709a6b521 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/d69ce46cde1e4316a3414f4709a6b521 2024-11-20T06:41:37,302 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/C of 88c2857bcc22b20889f07648453bfe35 into d69ce46cde1e4316a3414f4709a6b521(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:37,302 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:37,302 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/C, priority=13, startTime=1732084896451; duration=0sec 2024-11-20T06:41:37,302 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:37,302 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:C 2024-11-20T06:41:37,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084957314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084957314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084957324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084957324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084957324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,344 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/accc5ca640484cb0b884f1cb49a2d357 2024-11-20T06:41:37,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/e04d31d6b3b14842b8ce8ecba5be7a20 is 50, key is test_row_0/B:col10/1732084894793/Put/seqid=0 2024-11-20T06:41:37,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742212_1388 (size=12151) 2024-11-20T06:41:37,361 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/e04d31d6b3b14842b8ce8ecba5be7a20 2024-11-20T06:41:37,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/c19b59169a9944b1ade88a509a1ebd09 is 50, key is test_row_0/C:col10/1732084894793/Put/seqid=0 2024-11-20T06:41:37,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742213_1389 (size=12151) 2024-11-20T06:41:37,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084957633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084957634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084957644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084957644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084957644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:37,772 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/c19b59169a9944b1ade88a509a1ebd09 2024-11-20T06:41:37,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/accc5ca640484cb0b884f1cb49a2d357 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/accc5ca640484cb0b884f1cb49a2d357 2024-11-20T06:41:37,779 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/accc5ca640484cb0b884f1cb49a2d357, entries=150, sequenceid=154, filesize=30.4 K 2024-11-20T06:41:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/e04d31d6b3b14842b8ce8ecba5be7a20 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/e04d31d6b3b14842b8ce8ecba5be7a20 2024-11-20T06:41:37,784 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/e04d31d6b3b14842b8ce8ecba5be7a20, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T06:41:37,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/c19b59169a9944b1ade88a509a1ebd09 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/c19b59169a9944b1ade88a509a1ebd09 2024-11-20T06:41:37,790 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/c19b59169a9944b1ade88a509a1ebd09, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T06:41:37,791 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 88c2857bcc22b20889f07648453bfe35 in 1263ms, sequenceid=154, compaction requested=false 2024-11-20T06:41:37,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:37,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:37,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-20T06:41:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-20T06:41:37,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-20T06:41:37,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8760 sec 2024-11-20T06:41:37,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.9190 sec 2024-11-20T06:41:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T06:41:38,024 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:38,024 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-20T06:41:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T06:41:38,026 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:38,027 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:38,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T06:41:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:38,147 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:38,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:41:38,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dea606fb7d074c1eb9bc0b5543062502_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084898147/Put/seqid=0 2024-11-20T06:41:38,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742214_1390 (size=14794) 2024-11-20T06:41:38,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084958168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084958169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084958170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084958172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084958173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:38,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:38,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084958284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084958284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084958284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084958284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084958284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:38,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:38,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T06:41:38,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084958494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084958494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084958494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084958494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084958494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,559 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:38,562 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dea606fb7d074c1eb9bc0b5543062502_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dea606fb7d074c1eb9bc0b5543062502_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:38,563 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/4aad7e93611145b1b57a5da79684ce8f, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:38,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/4aad7e93611145b1b57a5da79684ce8f is 175, key is test_row_0/A:col10/1732084898147/Put/seqid=0 2024-11-20T06:41:38,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742215_1391 (size=39749) 2024-11-20T06:41:38,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T06:41:38,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:38,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:38,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084958814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084958814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084958814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084958814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084958814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:38,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:38,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:38,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:38,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:38,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/4aad7e93611145b1b57a5da79684ce8f 2024-11-20T06:41:38,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/462483d8bfe841499c905f03e11715e6 is 50, key is test_row_0/B:col10/1732084898147/Put/seqid=0 2024-11-20T06:41:38,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742216_1392 (size=12151) 2024-11-20T06:41:39,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:39,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:39,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T06:41:39,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:39,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:39,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084959324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:39,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084959324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:39,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084959324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:39,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:39,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:39,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084959334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:39,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084959334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:39,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:39,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/462483d8bfe841499c905f03e11715e6 2024-11-20T06:41:39,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/05f27659e71d48b7a61f8067a05917c5 is 50, key is test_row_0/C:col10/1732084898147/Put/seqid=0 2024-11-20T06:41:39,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742217_1393 (size=12151) 2024-11-20T06:41:39,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:39,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:39,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:39,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:39,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:39,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:39,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:39,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/05f27659e71d48b7a61f8067a05917c5 2024-11-20T06:41:39,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/4aad7e93611145b1b57a5da79684ce8f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4aad7e93611145b1b57a5da79684ce8f 2024-11-20T06:41:39,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4aad7e93611145b1b57a5da79684ce8f, entries=200, sequenceid=178, filesize=38.8 K 2024-11-20T06:41:39,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/462483d8bfe841499c905f03e11715e6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/462483d8bfe841499c905f03e11715e6 2024-11-20T06:41:39,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/462483d8bfe841499c905f03e11715e6, entries=150, sequenceid=178, filesize=11.9 K 2024-11-20T06:41:39,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/05f27659e71d48b7a61f8067a05917c5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/05f27659e71d48b7a61f8067a05917c5 2024-11-20T06:41:39,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/05f27659e71d48b7a61f8067a05917c5, entries=150, sequenceid=178, filesize=11.9 K 2024-11-20T06:41:39,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 88c2857bcc22b20889f07648453bfe35 in 1668ms, sequenceid=178, compaction requested=true 2024-11-20T06:41:39,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:39,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:39,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:39,815 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:39,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:39,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:39,815 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:39,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:39,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:39,816 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:39,816 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:39,816 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/B is initiating minor compaction (all files) 2024-11-20T06:41:39,816 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/A is initiating minor compaction (all files) 2024-11-20T06:41:39,816 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/B in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,816 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/A in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,816 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/91d1d434db01402d819c8e43013bc537, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/e04d31d6b3b14842b8ce8ecba5be7a20, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/462483d8bfe841499c905f03e11715e6] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=35.9 K 2024-11-20T06:41:39,816 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/e25191d92f444957adef2c109ab5ce8b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/accc5ca640484cb0b884f1cb49a2d357, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4aad7e93611145b1b57a5da79684ce8f] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=99.9 K 2024-11-20T06:41:39,816 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,816 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/e25191d92f444957adef2c109ab5ce8b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/accc5ca640484cb0b884f1cb49a2d357, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4aad7e93611145b1b57a5da79684ce8f] 2024-11-20T06:41:39,816 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 91d1d434db01402d819c8e43013bc537, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084893616 2024-11-20T06:41:39,816 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting e04d31d6b3b14842b8ce8ecba5be7a20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084894793 2024-11-20T06:41:39,816 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting e25191d92f444957adef2c109ab5ce8b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084893616 2024-11-20T06:41:39,817 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 462483d8bfe841499c905f03e11715e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732084896992 2024-11-20T06:41:39,817 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting accc5ca640484cb0b884f1cb49a2d357, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084894793 2024-11-20T06:41:39,817 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aad7e93611145b1b57a5da79684ce8f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732084896992 2024-11-20T06:41:39,823 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:39,824 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112057c0ef61a6614783b9445f1273b37f71_88c2857bcc22b20889f07648453bfe35 store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:39,824 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:39,825 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/78d2e0a222694874881995187181a346 is 50, key is test_row_0/B:col10/1732084898147/Put/seqid=0 2024-11-20T06:41:39,826 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112057c0ef61a6614783b9445f1273b37f71_88c2857bcc22b20889f07648453bfe35, store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:39,826 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112057c0ef61a6614783b9445f1273b37f71_88c2857bcc22b20889f07648453bfe35 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:39,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742218_1394 (size=12561) 2024-11-20T06:41:39,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742219_1395 (size=4469) 2024-11-20T06:41:39,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T06:41:39,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:39,859 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T06:41:39,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112026697ac030dd463c8a4ce1581fcd625c_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084898169/Put/seqid=0 2024-11-20T06:41:39,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742220_1396 (size=12304) 2024-11-20T06:41:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T06:41:40,234 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/78d2e0a222694874881995187181a346 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/78d2e0a222694874881995187181a346 2024-11-20T06:41:40,238 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/B of 88c2857bcc22b20889f07648453bfe35 into 78d2e0a222694874881995187181a346(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:40,238 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:40,238 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/B, priority=13, startTime=1732084899815; duration=0sec 2024-11-20T06:41:40,238 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:40,238 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:B 2024-11-20T06:41:40,238 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:40,239 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:40,239 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/C is initiating minor compaction (all files) 2024-11-20T06:41:40,239 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/C in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:40,239 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/d69ce46cde1e4316a3414f4709a6b521, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/c19b59169a9944b1ade88a509a1ebd09, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/05f27659e71d48b7a61f8067a05917c5] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=35.9 K 2024-11-20T06:41:40,239 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d69ce46cde1e4316a3414f4709a6b521, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732084893616 2024-11-20T06:41:40,240 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c19b59169a9944b1ade88a509a1ebd09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084894793 2024-11-20T06:41:40,240 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f27659e71d48b7a61f8067a05917c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732084896992 2024-11-20T06:41:40,241 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:40,241 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/f9c91083e1c64c569be9a5ce8d8c72a9 is 175, key is test_row_0/A:col10/1732084898147/Put/seqid=0 2024-11-20T06:41:40,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742221_1397 (size=31515) 2024-11-20T06:41:40,251 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:40,251 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/f4b0703bb17f4fe3a84d1fddaeeb535c is 50, key is test_row_0/C:col10/1732084898147/Put/seqid=0 2024-11-20T06:41:40,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742222_1398 (size=12561) 2024-11-20T06:41:40,267 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/f9c91083e1c64c569be9a5ce8d8c72a9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/f9c91083e1c64c569be9a5ce8d8c72a9 2024-11-20T06:41:40,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:40,270 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/A of 88c2857bcc22b20889f07648453bfe35 into f9c91083e1c64c569be9a5ce8d8c72a9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:40,270 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:40,270 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/A, priority=13, startTime=1732084899815; duration=0sec 2024-11-20T06:41:40,271 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:40,271 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:A 2024-11-20T06:41:40,272 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112026697ac030dd463c8a4ce1581fcd625c_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112026697ac030dd463c8a4ce1581fcd625c_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:40,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/435a8c25e9da490ea2194135105d5d3d, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:40,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/435a8c25e9da490ea2194135105d5d3d is 175, key is test_row_0/A:col10/1732084898169/Put/seqid=0 2024-11-20T06:41:40,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742223_1399 (size=31105) 2024-11-20T06:41:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:40,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:40,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084960363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084960370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084960371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084960372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084960373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084960484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084960484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084960484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084960484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084960484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,662 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/f4b0703bb17f4fe3a84d1fddaeeb535c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/f4b0703bb17f4fe3a84d1fddaeeb535c 2024-11-20T06:41:40,665 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/C of 88c2857bcc22b20889f07648453bfe35 into f4b0703bb17f4fe3a84d1fddaeeb535c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:40,665 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:40,665 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/C, priority=13, startTime=1732084899815; duration=0sec 2024-11-20T06:41:40,665 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:40,665 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:C 2024-11-20T06:41:40,677 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/435a8c25e9da490ea2194135105d5d3d 2024-11-20T06:41:40,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/a3fcfc9dbe134f3cbd5956322d911609 is 50, key is test_row_0/B:col10/1732084898169/Put/seqid=0 2024-11-20T06:41:40,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742224_1400 (size=12151) 2024-11-20T06:41:40,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084960694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084960694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084960694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084960694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084960694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084961014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:41,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084961014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084961014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:41,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084961014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084961014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,091 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/a3fcfc9dbe134f3cbd5956322d911609 2024-11-20T06:41:41,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/7ed2e7bf1def456cad70f877b56e664a is 50, key is test_row_0/C:col10/1732084898169/Put/seqid=0 2024-11-20T06:41:41,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742225_1401 (size=12151) 2024-11-20T06:41:41,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084961524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:41,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084961524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084961524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084961534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:41,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084961534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:41,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:41,548 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/7ed2e7bf1def456cad70f877b56e664a 2024-11-20T06:41:41,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/435a8c25e9da490ea2194135105d5d3d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/435a8c25e9da490ea2194135105d5d3d 2024-11-20T06:41:41,554 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/435a8c25e9da490ea2194135105d5d3d, entries=150, sequenceid=193, filesize=30.4 K 2024-11-20T06:41:41,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/a3fcfc9dbe134f3cbd5956322d911609 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a3fcfc9dbe134f3cbd5956322d911609 2024-11-20T06:41:41,558 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a3fcfc9dbe134f3cbd5956322d911609, entries=150, sequenceid=193, filesize=11.9 K 2024-11-20T06:41:41,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/7ed2e7bf1def456cad70f877b56e664a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/7ed2e7bf1def456cad70f877b56e664a 2024-11-20T06:41:41,561 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/7ed2e7bf1def456cad70f877b56e664a, entries=150, sequenceid=193, filesize=11.9 K 2024-11-20T06:41:41,562 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 88c2857bcc22b20889f07648453bfe35 in 1703ms, sequenceid=193, compaction requested=false 2024-11-20T06:41:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-20T06:41:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-20T06:41:41,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-20T06:41:41,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5370 sec 2024-11-20T06:41:41,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.5410 sec 2024-11-20T06:41:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T06:41:42,184 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:42,185 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-20T06:41:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T06:41:42,186 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:42,187 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:42,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:42,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T06:41:42,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T06:41:42,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:42,339 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T06:41:42,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b98c623f8acf45ec89084690a2158253_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084900359/Put/seqid=0 2024-11-20T06:41:42,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742226_1402 (size=12304) 2024-11-20T06:41:42,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:42,350 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b98c623f8acf45ec89084690a2158253_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b98c623f8acf45ec89084690a2158253_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:42,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/0b948fa3931342a28519975a99d61ba4, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:42,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/0b948fa3931342a28519975a99d61ba4 is 175, key is test_row_0/A:col10/1732084900359/Put/seqid=0 2024-11-20T06:41:42,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742227_1403 (size=31105) 2024-11-20T06:41:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T06:41:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:42,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:42,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084962554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084962557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,567 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 88c2857bcc22b20889f07648453bfe35 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:41:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084962561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084962567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084962569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084962673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084962673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084962683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084962684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,756 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/0b948fa3931342a28519975a99d61ba4 2024-11-20T06:41:42,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/4699c307cb144bcca166cf87e20d3443 is 50, key is test_row_0/B:col10/1732084900359/Put/seqid=0 2024-11-20T06:41:42,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742228_1404 (size=12151) 2024-11-20T06:41:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T06:41:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084962883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084962883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084962894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084962894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,164 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/4699c307cb144bcca166cf87e20d3443 2024-11-20T06:41:43,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/dbf8e4cacd6444639bd99ecb1f7a6faf is 50, key is test_row_0/C:col10/1732084900359/Put/seqid=0 2024-11-20T06:41:43,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742229_1405 (size=12151) 2024-11-20T06:41:43,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084963204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084963204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084963213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084963214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T06:41:43,577 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/dbf8e4cacd6444639bd99ecb1f7a6faf 2024-11-20T06:41:43,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/0b948fa3931342a28519975a99d61ba4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/0b948fa3931342a28519975a99d61ba4 2024-11-20T06:41:43,583 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/0b948fa3931342a28519975a99d61ba4, entries=150, sequenceid=217, filesize=30.4 K 2024-11-20T06:41:43,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/4699c307cb144bcca166cf87e20d3443 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/4699c307cb144bcca166cf87e20d3443 2024-11-20T06:41:43,587 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/4699c307cb144bcca166cf87e20d3443, entries=150, sequenceid=217, filesize=11.9 K 2024-11-20T06:41:43,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/dbf8e4cacd6444639bd99ecb1f7a6faf as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/dbf8e4cacd6444639bd99ecb1f7a6faf 2024-11-20T06:41:43,591 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/dbf8e4cacd6444639bd99ecb1f7a6faf, entries=150, sequenceid=217, filesize=11.9 K 2024-11-20T06:41:43,591 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 88c2857bcc22b20889f07648453bfe35 in 1252ms, sequenceid=217, compaction requested=true 2024-11-20T06:41:43,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:43,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:43,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-20T06:41:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-20T06:41:43,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-20T06:41:43,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4060 sec 2024-11-20T06:41:43,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4100 sec 2024-11-20T06:41:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:43,721 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:43,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T06:41:43,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209f40d69b32ff4d6a82c29c93655ff4c1_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084902566/Put/seqid=0 2024-11-20T06:41:43,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742230_1406 (size=14794) 2024-11-20T06:41:43,733 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:43,736 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209f40d69b32ff4d6a82c29c93655ff4c1_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209f40d69b32ff4d6a82c29c93655ff4c1_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:43,737 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/be7b55136a5d41dfaf220a604c04b9d2, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:43,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/be7b55136a5d41dfaf220a604c04b9d2 is 175, key is test_row_0/A:col10/1732084902566/Put/seqid=0 2024-11-20T06:41:43,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742231_1407 (size=39749) 2024-11-20T06:41:43,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084963756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084963757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084963759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084963765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084963874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084963874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084963874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084963874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084964084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084964084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084964084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084964084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,140 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/be7b55136a5d41dfaf220a604c04b9d2 2024-11-20T06:41:44,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/8e9cf989f7514e6b9848a3fd5b9757dd is 50, key is test_row_0/B:col10/1732084902566/Put/seqid=0 2024-11-20T06:41:44,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742232_1408 (size=12151) 2024-11-20T06:41:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T06:41:44,334 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:44,335 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-20T06:41:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T06:41:44,336 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:44,336 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:44,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084964404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084964404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084964404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084964404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T06:41:44,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T06:41:44,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/8e9cf989f7514e6b9848a3fd5b9757dd 2024-11-20T06:41:44,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/36bd78542f8d43e9b89bf87a9b9a18b9 is 50, key is test_row_0/C:col10/1732084902566/Put/seqid=0 2024-11-20T06:41:44,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742233_1409 (size=12151) 2024-11-20T06:41:44,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/36bd78542f8d43e9b89bf87a9b9a18b9 2024-11-20T06:41:44,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/be7b55136a5d41dfaf220a604c04b9d2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/be7b55136a5d41dfaf220a604c04b9d2 2024-11-20T06:41:44,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/be7b55136a5d41dfaf220a604c04b9d2, entries=200, sequenceid=234, filesize=38.8 K 2024-11-20T06:41:44,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/8e9cf989f7514e6b9848a3fd5b9757dd as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/8e9cf989f7514e6b9848a3fd5b9757dd 2024-11-20T06:41:44,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/8e9cf989f7514e6b9848a3fd5b9757dd, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T06:41:44,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/36bd78542f8d43e9b89bf87a9b9a18b9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/36bd78542f8d43e9b89bf87a9b9a18b9 2024-11-20T06:41:44,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/36bd78542f8d43e9b89bf87a9b9a18b9, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T06:41:44,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 88c2857bcc22b20889f07648453bfe35 in 861ms, sequenceid=234, compaction requested=true 2024-11-20T06:41:44,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:44,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:44,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:44,583 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:41:44,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:44,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:44,583 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:41:44,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:44,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:44,584 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:41:44,584 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:41:44,584 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/A is initiating minor compaction (all files) 2024-11-20T06:41:44,584 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/B is initiating minor compaction (all files) 2024-11-20T06:41:44,584 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/A in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,584 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/B in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,584 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/f9c91083e1c64c569be9a5ce8d8c72a9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/435a8c25e9da490ea2194135105d5d3d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/0b948fa3931342a28519975a99d61ba4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/be7b55136a5d41dfaf220a604c04b9d2] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=130.3 K 2024-11-20T06:41:44,584 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/78d2e0a222694874881995187181a346, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a3fcfc9dbe134f3cbd5956322d911609, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/4699c307cb144bcca166cf87e20d3443, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/8e9cf989f7514e6b9848a3fd5b9757dd] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=47.9 K 2024-11-20T06:41:44,584 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,584 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/f9c91083e1c64c569be9a5ce8d8c72a9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/435a8c25e9da490ea2194135105d5d3d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/0b948fa3931342a28519975a99d61ba4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/be7b55136a5d41dfaf220a604c04b9d2] 2024-11-20T06:41:44,584 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 78d2e0a222694874881995187181a346, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732084896992 2024-11-20T06:41:44,585 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9c91083e1c64c569be9a5ce8d8c72a9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732084896992 2024-11-20T06:41:44,585 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a3fcfc9dbe134f3cbd5956322d911609, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732084898167 2024-11-20T06:41:44,585 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 435a8c25e9da490ea2194135105d5d3d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732084898167 2024-11-20T06:41:44,585 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 4699c307cb144bcca166cf87e20d3443, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732084900359 2024-11-20T06:41:44,585 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b948fa3931342a28519975a99d61ba4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732084900359 2024-11-20T06:41:44,585 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e9cf989f7514e6b9848a3fd5b9757dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732084902559 2024-11-20T06:41:44,585 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting be7b55136a5d41dfaf220a604c04b9d2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732084902559 2024-11-20T06:41:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:44,590 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:44,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:41:44,594 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:44,594 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:44,594 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/cae0994d336f4e668bc96d87614ffa88 is 50, key is test_row_0/B:col10/1732084902566/Put/seqid=0 2024-11-20T06:41:44,597 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411200e12b0de658d404d9221fe733f1fc93e_88c2857bcc22b20889f07648453bfe35 store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:44,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742234_1410 (size=12697) 2024-11-20T06:41:44,600 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411200e12b0de658d404d9221fe733f1fc93e_88c2857bcc22b20889f07648453bfe35, store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:44,600 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200e12b0de658d404d9221fe733f1fc93e_88c2857bcc22b20889f07648453bfe35 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:44,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120745352bc36b1421bbb1564de048ef564_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084903754/Put/seqid=0 2024-11-20T06:41:44,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742236_1412 (size=14794) 2024-11-20T06:41:44,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742235_1411 (size=4469) 2024-11-20T06:41:44,609 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#A#compaction#352 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:44,610 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/9f3178f150bf4aa783b441f97544c63d is 175, key is test_row_0/A:col10/1732084902566/Put/seqid=0 2024-11-20T06:41:44,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742237_1413 (size=31651) 2024-11-20T06:41:44,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T06:41:44,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:44,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T06:41:44,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084964698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T06:41:44,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084964814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084964913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084964924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084964924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084964924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:44,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T06:41:44,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:44,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:44,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:44,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T06:41:45,003 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/cae0994d336f4e668bc96d87614ffa88 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/cae0994d336f4e668bc96d87614ffa88 2024-11-20T06:41:45,007 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/B of 88c2857bcc22b20889f07648453bfe35 into cae0994d336f4e668bc96d87614ffa88(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:45,007 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:45,007 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/B, priority=12, startTime=1732084904583; duration=0sec 2024-11-20T06:41:45,008 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:45,008 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:B 2024-11-20T06:41:45,008 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:45,008 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:41:45,009 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:41:45,009 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 88c2857bcc22b20889f07648453bfe35/C is initiating minor compaction (all files) 2024-11-20T06:41:45,009 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88c2857bcc22b20889f07648453bfe35/C in TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:45,009 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/f4b0703bb17f4fe3a84d1fddaeeb535c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/7ed2e7bf1def456cad70f877b56e664a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/dbf8e4cacd6444639bd99ecb1f7a6faf, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/36bd78542f8d43e9b89bf87a9b9a18b9] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp, totalSize=47.9 K 2024-11-20T06:41:45,009 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting f4b0703bb17f4fe3a84d1fddaeeb535c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732084896992 2024-11-20T06:41:45,010 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed2e7bf1def456cad70f877b56e664a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732084898167 2024-11-20T06:41:45,010 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting dbf8e4cacd6444639bd99ecb1f7a6faf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732084900359 2024-11-20T06:41:45,010 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 36bd78542f8d43e9b89bf87a9b9a18b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732084902559 2024-11-20T06:41:45,011 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120745352bc36b1421bbb1564de048ef564_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120745352bc36b1421bbb1564de048ef564_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:45,011 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/ba23ac27b2a846c587919c88d44e8636, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:45,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/ba23ac27b2a846c587919c88d44e8636 is 175, key is test_row_0/A:col10/1732084903754/Put/seqid=0 2024-11-20T06:41:45,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742238_1414 (size=39749) 2024-11-20T06:41:45,018 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88c2857bcc22b20889f07648453bfe35#C#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:45,018 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/ba23ac27b2a846c587919c88d44e8636 2024-11-20T06:41:45,019 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/a3b5c1f669fc4eeea567135731f1b4e7 is 50, key is test_row_0/C:col10/1732084902566/Put/seqid=0 2024-11-20T06:41:45,020 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/9f3178f150bf4aa783b441f97544c63d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/9f3178f150bf4aa783b441f97544c63d 2024-11-20T06:41:45,024 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/A of 88c2857bcc22b20889f07648453bfe35 into 9f3178f150bf4aa783b441f97544c63d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:45,024 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:45,024 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/A, priority=12, startTime=1732084904582; duration=0sec 2024-11-20T06:41:45,025 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:45,025 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:A 2024-11-20T06:41:45,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742239_1415 (size=12697) 2024-11-20T06:41:45,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d41238ba7dfb4eeb93208b00292ad829 is 50, key is test_row_0/B:col10/1732084903754/Put/seqid=0 2024-11-20T06:41:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742240_1416 (size=12151) 2024-11-20T06:41:45,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084965024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:45,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T06:41:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:45,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:45,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T06:41:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:45,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:45,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084965343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:45,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T06:41:45,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:45,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:45,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:45,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:45,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:45,429 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/a3b5c1f669fc4eeea567135731f1b4e7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/a3b5c1f669fc4eeea567135731f1b4e7 2024-11-20T06:41:45,433 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88c2857bcc22b20889f07648453bfe35/C of 88c2857bcc22b20889f07648453bfe35 into a3b5c1f669fc4eeea567135731f1b4e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:45,433 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:45,433 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., storeName=88c2857bcc22b20889f07648453bfe35/C, priority=12, startTime=1732084904583; duration=0sec 2024-11-20T06:41:45,433 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:45,433 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:C 2024-11-20T06:41:45,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d41238ba7dfb4eeb93208b00292ad829 2024-11-20T06:41:45,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/cae67dc8e28847c98a82778b2f2e0ebe is 50, key is test_row_0/C:col10/1732084903754/Put/seqid=0 2024-11-20T06:41:45,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742241_1417 (size=12151) 2024-11-20T06:41:45,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/cae67dc8e28847c98a82778b2f2e0ebe 2024-11-20T06:41:45,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/ba23ac27b2a846c587919c88d44e8636 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/ba23ac27b2a846c587919c88d44e8636 2024-11-20T06:41:45,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/ba23ac27b2a846c587919c88d44e8636, entries=200, sequenceid=254, filesize=38.8 K 2024-11-20T06:41:45,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d41238ba7dfb4eeb93208b00292ad829 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d41238ba7dfb4eeb93208b00292ad829 2024-11-20T06:41:45,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d41238ba7dfb4eeb93208b00292ad829, entries=150, sequenceid=254, filesize=11.9 K 2024-11-20T06:41:45,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/cae67dc8e28847c98a82778b2f2e0ebe as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/cae67dc8e28847c98a82778b2f2e0ebe 2024-11-20T06:41:45,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/cae67dc8e28847c98a82778b2f2e0ebe, entries=150, sequenceid=254, filesize=11.9 K 2024-11-20T06:41:45,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 88c2857bcc22b20889f07648453bfe35 in 875ms, sequenceid=254, compaction requested=false 2024-11-20T06:41:45,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T06:41:45,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T06:41:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:45,557 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T06:41:45,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202bc6cfbe89814cf38296dd0457d5fb1d_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084904696/Put/seqid=0 2024-11-20T06:41:45,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742242_1418 (size=12454) 2024-11-20T06:41:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:45,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. as already flushing 2024-11-20T06:41:45,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084965947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084965951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1732084965953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39822 deadline: 1732084965953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:45,964 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 88c2857bcc22b20889f07648453bfe35 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:41:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,964 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 88c2857bcc22b20889f07648453bfe35 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:41:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:45,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084965961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:45,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:45,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:45,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:45,973 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202bc6cfbe89814cf38296dd0457d5fb1d_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202bc6cfbe89814cf38296dd0457d5fb1d_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:45,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6126808ce6cc43118b620a14e5c0b8b8, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:45,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6126808ce6cc43118b620a14e5c0b8b8 is 175, key is test_row_0/A:col10/1732084904696/Put/seqid=0 2024-11-20T06:41:45,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742243_1419 (size=31255) 2024-11-20T06:41:46,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084966063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084966074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:46,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084966074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084966278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:46,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084966284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:46,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084966294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:46,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:46,378 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6126808ce6cc43118b620a14e5c0b8b8 2024-11-20T06:41:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/3c1e2cf506774e229b2fb19f1c53910e is 50, key is test_row_0/B:col10/1732084904696/Put/seqid=0 2024-11-20T06:41:46,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742244_1420 (size=12301) 2024-11-20T06:41:46,438 INFO [Thread-1581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1581. 2024-11-20T06:41:46,438 DEBUG [Thread-1581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:46,438 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:46,438 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:46,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:46,438 INFO [Thread-1583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1583. 2024-11-20T06:41:46,438 DEBUG [Thread-1583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:46,438 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:46,438 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:46,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:46,441 INFO [Thread-1579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1579. 2024-11-20T06:41:46,441 DEBUG [Thread-1579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:46,441 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:46,441 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:46,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:46,447 INFO [Thread-1575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1575. 2024-11-20T06:41:46,447 DEBUG [Thread-1575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:46,447 DEBUG [Thread-1575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:46,447 DEBUG [Thread-1575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:46,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:46,447 INFO [Thread-1577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1577. 2024-11-20T06:41:46,447 DEBUG [Thread-1577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:46,447 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:46,447 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:46,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T06:41:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084966595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084966595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:46,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084966604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:46,825 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/3c1e2cf506774e229b2fb19f1c53910e 2024-11-20T06:41:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/1825185f09824ffd8d53573cd1fafd60 is 50, key is test_row_0/C:col10/1732084904696/Put/seqid=0 2024-11-20T06:41:46,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742245_1421 (size=12301) 2024-11-20T06:41:47,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:47,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39764 deadline: 1732084967105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39808 deadline: 1732084967104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:47,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:47,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39806 deadline: 1732084967114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:41:47,243 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/1825185f09824ffd8d53573cd1fafd60 2024-11-20T06:41:47,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/6126808ce6cc43118b620a14e5c0b8b8 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6126808ce6cc43118b620a14e5c0b8b8 2024-11-20T06:41:47,251 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6126808ce6cc43118b620a14e5c0b8b8, entries=150, sequenceid=273, filesize=30.5 K 2024-11-20T06:41:47,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/3c1e2cf506774e229b2fb19f1c53910e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/3c1e2cf506774e229b2fb19f1c53910e 2024-11-20T06:41:47,255 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/3c1e2cf506774e229b2fb19f1c53910e, entries=150, sequenceid=273, filesize=12.0 K 2024-11-20T06:41:47,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/1825185f09824ffd8d53573cd1fafd60 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/1825185f09824ffd8d53573cd1fafd60 2024-11-20T06:41:47,259 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/1825185f09824ffd8d53573cd1fafd60, entries=150, sequenceid=273, filesize=12.0 K 2024-11-20T06:41:47,260 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 88c2857bcc22b20889f07648453bfe35 in 1702ms, sequenceid=273, compaction requested=true 2024-11-20T06:41:47,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:47,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:47,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-20T06:41:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-20T06:41:47,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-20T06:41:47,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9240 sec 2024-11-20T06:41:47,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.9280 sec 2024-11-20T06:41:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:47,987 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:47,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T06:41:47,987 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-11-20T06:41:47,987 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-11-20T06:41:47,987 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:47,987 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:47,987 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:47,987 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:47,987 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:47,987 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:47,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:47,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:47,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120927f13cf798040a5bd9ad5ecda9209e8_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084905930/Put/seqid=0 2024-11-20T06:41:47,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742246_1422 (size=12454) 2024-11-20T06:41:48,117 INFO [Thread-1570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1570. 2024-11-20T06:41:48,117 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-11-20T06:41:48,117 DEBUG [Thread-1570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:48,117 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:48,117 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:48,117 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:48,117 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:48,118 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:48,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:48,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:48,126 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-11-20T06:41:48,126 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:41:48,126 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:48,126 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:48,126 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:48,401 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:48,411 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120927f13cf798040a5bd9ad5ecda9209e8_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120927f13cf798040a5bd9ad5ecda9209e8_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:48,413 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/9b03c9dc17d14880b870014a781ca51f, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:48,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/9b03c9dc17d14880b870014a781ca51f is 175, key is test_row_0/A:col10/1732084905930/Put/seqid=0 2024-11-20T06:41:48,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742247_1423 (size=31255) 2024-11-20T06:41:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T06:41:48,504 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:48,504 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1508 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4513 rows 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1506 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4518 rows 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1504 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4504 rows 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1505 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4498 rows 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1503 2024-11-20T06:41:48,505 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4509 rows 2024-11-20T06:41:48,505 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:41:48,505 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:41:48,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:48,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:48,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:41:48,506 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T06:41:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T06:41:48,510 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084908510"}]},"ts":"1732084908510"} 2024-11-20T06:41:48,511 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T06:41:48,512 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T06:41:48,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T06:41:48,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, UNASSIGN}] 2024-11-20T06:41:48,515 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, UNASSIGN 2024-11-20T06:41:48,516 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=88c2857bcc22b20889f07648453bfe35, regionState=CLOSING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:48,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, UNASSIGN because future has completed 2024-11-20T06:41:48,518 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T06:41:48,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:41:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T06:41:48,671 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:48,671 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T06:41:48,672 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 88c2857bcc22b20889f07648453bfe35, disabling compactions & flushes 2024-11-20T06:41:48,672 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:48,819 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/9b03c9dc17d14880b870014a781ca51f 2024-11-20T06:41:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T06:41:48,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d333ed1b316e403aabf71b8486c81bb9 is 50, key is test_row_0/B:col10/1732084905930/Put/seqid=0 2024-11-20T06:41:48,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742248_1424 (size=12301) 2024-11-20T06:41:49,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T06:41:49,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d333ed1b316e403aabf71b8486c81bb9 2024-11-20T06:41:49,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/e71da35248b94c98b02fb1a96faf2969 is 50, key is test_row_0/C:col10/1732084905930/Put/seqid=0 2024-11-20T06:41:49,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742249_1425 (size=12301) 2024-11-20T06:41:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T06:41:49,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/e71da35248b94c98b02fb1a96faf2969 2024-11-20T06:41:49,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/9b03c9dc17d14880b870014a781ca51f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/9b03c9dc17d14880b870014a781ca51f 2024-11-20T06:41:49,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/9b03c9dc17d14880b870014a781ca51f, entries=150, sequenceid=295, filesize=30.5 K 2024-11-20T06:41:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/d333ed1b316e403aabf71b8486c81bb9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d333ed1b316e403aabf71b8486c81bb9 2024-11-20T06:41:49,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d333ed1b316e403aabf71b8486c81bb9, entries=150, sequenceid=295, filesize=12.0 K 2024-11-20T06:41:49,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/e71da35248b94c98b02fb1a96faf2969 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/e71da35248b94c98b02fb1a96faf2969 2024-11-20T06:41:49,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/e71da35248b94c98b02fb1a96faf2969, entries=150, sequenceid=295, filesize=12.0 K 2024-11-20T06:41:49,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=20.13 KB/20610 for 88c2857bcc22b20889f07648453bfe35 in 1688ms, sequenceid=295, compaction requested=true 2024-11-20T06:41:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88c2857bcc22b20889f07648453bfe35: 2024-11-20T06:41:49,675 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:49,675 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:49,675 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. after waiting 0 ms 2024-11-20T06:41:49,675 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88c2857bcc22b20889f07648453bfe35:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:49,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:49,675 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 88c2857bcc22b20889f07648453bfe35 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-20T06:41:49,675 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. because compaction request was cancelled 2024-11-20T06:41:49,675 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. because compaction request was cancelled 2024-11-20T06:41:49,675 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:A 2024-11-20T06:41:49,675 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:B 2024-11-20T06:41:49,675 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. because compaction request was cancelled 2024-11-20T06:41:49,675 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88c2857bcc22b20889f07648453bfe35:C 2024-11-20T06:41:49,679 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a6ccf02ff8e4420a959a717e80981e04_88c2857bcc22b20889f07648453bfe35 is 50, key is test_row_0/A:col10/1732084908124/Put/seqid=0 2024-11-20T06:41:49,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742250_1426 (size=9914) 2024-11-20T06:41:50,082 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:50,086 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a6ccf02ff8e4420a959a717e80981e04_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a6ccf02ff8e4420a959a717e80981e04_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:50,087 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/c1f5e4d49e1a4aa19f0e02eceb491f5c, store: [table=TestAcidGuarantees family=A region=88c2857bcc22b20889f07648453bfe35] 2024-11-20T06:41:50,087 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/c1f5e4d49e1a4aa19f0e02eceb491f5c is 175, key is test_row_0/A:col10/1732084908124/Put/seqid=0 2024-11-20T06:41:50,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742251_1427 (size=22561) 2024-11-20T06:41:50,090 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/c1f5e4d49e1a4aa19f0e02eceb491f5c 2024-11-20T06:41:50,095 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/de1517633fe844e796dd81fa07818405 is 50, key is test_row_0/B:col10/1732084908124/Put/seqid=0 2024-11-20T06:41:50,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742252_1428 (size=9857) 2024-11-20T06:41:50,499 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/de1517633fe844e796dd81fa07818405 2024-11-20T06:41:50,507 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/bb06f3d7516e4afb8ecdedfada9c2c44 is 50, key is test_row_0/C:col10/1732084908124/Put/seqid=0 2024-11-20T06:41:50,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742253_1429 (size=9857) 2024-11-20T06:41:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T06:41:50,914 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/bb06f3d7516e4afb8ecdedfada9c2c44 2024-11-20T06:41:50,923 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/A/c1f5e4d49e1a4aa19f0e02eceb491f5c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/c1f5e4d49e1a4aa19f0e02eceb491f5c 2024-11-20T06:41:50,927 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/c1f5e4d49e1a4aa19f0e02eceb491f5c, entries=100, sequenceid=301, filesize=22.0 K 2024-11-20T06:41:50,928 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/B/de1517633fe844e796dd81fa07818405 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/de1517633fe844e796dd81fa07818405 2024-11-20T06:41:50,931 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/de1517633fe844e796dd81fa07818405, entries=100, sequenceid=301, filesize=9.6 K 2024-11-20T06:41:50,931 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/.tmp/C/bb06f3d7516e4afb8ecdedfada9c2c44 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/bb06f3d7516e4afb8ecdedfada9c2c44 2024-11-20T06:41:50,934 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/bb06f3d7516e4afb8ecdedfada9c2c44, entries=100, sequenceid=301, filesize=9.6 K 2024-11-20T06:41:50,934 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 88c2857bcc22b20889f07648453bfe35 in 1259ms, sequenceid=301, compaction requested=true 2024-11-20T06:41:50,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/943e657785c7457eb4c2fc0728dfdd55, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6e981b2c9c1449098ba6ffadd97d9aef, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6c5380bf170b495cb0d2744cd111e09c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4ede75d69b9d45089eb1db5d07a69100, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/2894173cc86644959f5f8500e2445345, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/69d6556c6b7d4cb987009b7f34416861, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/ca5345cb95e84d6f97e20cc1226a0af4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/a1e945e085aa4f9395cc99c23141cd24, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/3ece0a0d1fad4a34be13cd515e269708, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/e25191d92f444957adef2c109ab5ce8b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/accc5ca640484cb0b884f1cb49a2d357, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4aad7e93611145b1b57a5da79684ce8f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/f9c91083e1c64c569be9a5ce8d8c72a9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/435a8c25e9da490ea2194135105d5d3d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/0b948fa3931342a28519975a99d61ba4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/be7b55136a5d41dfaf220a604c04b9d2] to archive 2024-11-20T06:41:50,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:41:50,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/943e657785c7457eb4c2fc0728dfdd55 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/943e657785c7457eb4c2fc0728dfdd55 2024-11-20T06:41:50,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6e981b2c9c1449098ba6ffadd97d9aef to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6e981b2c9c1449098ba6ffadd97d9aef 2024-11-20T06:41:50,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6c5380bf170b495cb0d2744cd111e09c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6c5380bf170b495cb0d2744cd111e09c 2024-11-20T06:41:50,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4ede75d69b9d45089eb1db5d07a69100 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4ede75d69b9d45089eb1db5d07a69100 2024-11-20T06:41:50,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/2894173cc86644959f5f8500e2445345 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/2894173cc86644959f5f8500e2445345 2024-11-20T06:41:50,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/69d6556c6b7d4cb987009b7f34416861 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/69d6556c6b7d4cb987009b7f34416861 2024-11-20T06:41:50,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/ca5345cb95e84d6f97e20cc1226a0af4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/ca5345cb95e84d6f97e20cc1226a0af4 2024-11-20T06:41:50,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/a1e945e085aa4f9395cc99c23141cd24 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/a1e945e085aa4f9395cc99c23141cd24 2024-11-20T06:41:50,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/3ece0a0d1fad4a34be13cd515e269708 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/3ece0a0d1fad4a34be13cd515e269708 2024-11-20T06:41:50,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/e25191d92f444957adef2c109ab5ce8b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/e25191d92f444957adef2c109ab5ce8b 2024-11-20T06:41:50,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/accc5ca640484cb0b884f1cb49a2d357 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/accc5ca640484cb0b884f1cb49a2d357 2024-11-20T06:41:50,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4aad7e93611145b1b57a5da79684ce8f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/4aad7e93611145b1b57a5da79684ce8f 2024-11-20T06:41:50,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/f9c91083e1c64c569be9a5ce8d8c72a9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/f9c91083e1c64c569be9a5ce8d8c72a9 2024-11-20T06:41:50,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/435a8c25e9da490ea2194135105d5d3d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/435a8c25e9da490ea2194135105d5d3d 2024-11-20T06:41:50,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/0b948fa3931342a28519975a99d61ba4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/0b948fa3931342a28519975a99d61ba4 2024-11-20T06:41:50,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/be7b55136a5d41dfaf220a604c04b9d2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/be7b55136a5d41dfaf220a604c04b9d2 2024-11-20T06:41:50,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/03287615e237473a8888665112ffc747, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/127d7be71aac427eba3d20a810255cc7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/fe3abab9d8924cb0ae5ebd2c2f720532, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/b1cf06610fc5445abe37b3d86e1b116b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a9863e2afbaa488fae1c7562a1562bae, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d9c578bb5bf74bb291b56395cd9a2aa4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/adee63115aff4100b345177a343d9b1b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/ca7d2fcc6cfe4b7aa492d40a236a4fba, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/91d1d434db01402d819c8e43013bc537, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d3514a5cfb664f348648c43fe97ba2fc, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/e04d31d6b3b14842b8ce8ecba5be7a20, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/78d2e0a222694874881995187181a346, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/462483d8bfe841499c905f03e11715e6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a3fcfc9dbe134f3cbd5956322d911609, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/4699c307cb144bcca166cf87e20d3443, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/8e9cf989f7514e6b9848a3fd5b9757dd] to archive 2024-11-20T06:41:50,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:41:50,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/03287615e237473a8888665112ffc747 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/03287615e237473a8888665112ffc747 2024-11-20T06:41:50,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/127d7be71aac427eba3d20a810255cc7 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/127d7be71aac427eba3d20a810255cc7 2024-11-20T06:41:50,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/fe3abab9d8924cb0ae5ebd2c2f720532 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/fe3abab9d8924cb0ae5ebd2c2f720532 2024-11-20T06:41:50,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/b1cf06610fc5445abe37b3d86e1b116b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/b1cf06610fc5445abe37b3d86e1b116b 2024-11-20T06:41:50,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a9863e2afbaa488fae1c7562a1562bae to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a9863e2afbaa488fae1c7562a1562bae 2024-11-20T06:41:50,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d9c578bb5bf74bb291b56395cd9a2aa4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d9c578bb5bf74bb291b56395cd9a2aa4 2024-11-20T06:41:50,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/adee63115aff4100b345177a343d9b1b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/adee63115aff4100b345177a343d9b1b 2024-11-20T06:41:50,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/ca7d2fcc6cfe4b7aa492d40a236a4fba to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/ca7d2fcc6cfe4b7aa492d40a236a4fba 2024-11-20T06:41:50,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/91d1d434db01402d819c8e43013bc537 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/91d1d434db01402d819c8e43013bc537 2024-11-20T06:41:50,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d3514a5cfb664f348648c43fe97ba2fc to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d3514a5cfb664f348648c43fe97ba2fc 2024-11-20T06:41:50,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/e04d31d6b3b14842b8ce8ecba5be7a20 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/e04d31d6b3b14842b8ce8ecba5be7a20 2024-11-20T06:41:50,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/78d2e0a222694874881995187181a346 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/78d2e0a222694874881995187181a346 2024-11-20T06:41:50,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/462483d8bfe841499c905f03e11715e6 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/462483d8bfe841499c905f03e11715e6 2024-11-20T06:41:50,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a3fcfc9dbe134f3cbd5956322d911609 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/a3fcfc9dbe134f3cbd5956322d911609 2024-11-20T06:41:50,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/4699c307cb144bcca166cf87e20d3443 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/4699c307cb144bcca166cf87e20d3443 2024-11-20T06:41:50,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/8e9cf989f7514e6b9848a3fd5b9757dd to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/8e9cf989f7514e6b9848a3fd5b9757dd 2024-11-20T06:41:50,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/cf854552520d43c8979bc5800cfbb1dd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/daec66f8766e4ec78ebf210cad3575e7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/8ea59497a34f459dbaabc8f5d615e7d0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/11fc8da624f44b998c78c94312ef9800, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/e928a5cd27e842709d2733502caaff79, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5471994daf7e425186ed1b92d8582169, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5296e1514a954e859fbee9b32d732583, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/9deec385e4c0401eaf3e97d38f35f49e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/d69ce46cde1e4316a3414f4709a6b521, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/54a1dbd8fe8b4de7a7cde2689d49938e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/c19b59169a9944b1ade88a509a1ebd09, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/f4b0703bb17f4fe3a84d1fddaeeb535c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/05f27659e71d48b7a61f8067a05917c5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/7ed2e7bf1def456cad70f877b56e664a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/dbf8e4cacd6444639bd99ecb1f7a6faf, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/36bd78542f8d43e9b89bf87a9b9a18b9] to archive 2024-11-20T06:41:50,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:41:50,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/cf854552520d43c8979bc5800cfbb1dd to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/cf854552520d43c8979bc5800cfbb1dd 2024-11-20T06:41:50,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/daec66f8766e4ec78ebf210cad3575e7 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/daec66f8766e4ec78ebf210cad3575e7 2024-11-20T06:41:50,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/8ea59497a34f459dbaabc8f5d615e7d0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/8ea59497a34f459dbaabc8f5d615e7d0 2024-11-20T06:41:50,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/11fc8da624f44b998c78c94312ef9800 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/11fc8da624f44b998c78c94312ef9800 2024-11-20T06:41:50,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/e928a5cd27e842709d2733502caaff79 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/e928a5cd27e842709d2733502caaff79 2024-11-20T06:41:50,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5471994daf7e425186ed1b92d8582169 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5471994daf7e425186ed1b92d8582169 2024-11-20T06:41:50,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5296e1514a954e859fbee9b32d732583 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/5296e1514a954e859fbee9b32d732583 2024-11-20T06:41:50,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/9deec385e4c0401eaf3e97d38f35f49e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/9deec385e4c0401eaf3e97d38f35f49e 2024-11-20T06:41:50,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/d69ce46cde1e4316a3414f4709a6b521 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/d69ce46cde1e4316a3414f4709a6b521 2024-11-20T06:41:50,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/54a1dbd8fe8b4de7a7cde2689d49938e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/54a1dbd8fe8b4de7a7cde2689d49938e 2024-11-20T06:41:50,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/c19b59169a9944b1ade88a509a1ebd09 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/c19b59169a9944b1ade88a509a1ebd09 2024-11-20T06:41:50,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/f4b0703bb17f4fe3a84d1fddaeeb535c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/f4b0703bb17f4fe3a84d1fddaeeb535c 2024-11-20T06:41:50,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/05f27659e71d48b7a61f8067a05917c5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/05f27659e71d48b7a61f8067a05917c5 2024-11-20T06:41:50,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/7ed2e7bf1def456cad70f877b56e664a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/7ed2e7bf1def456cad70f877b56e664a 2024-11-20T06:41:50,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/dbf8e4cacd6444639bd99ecb1f7a6faf to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/dbf8e4cacd6444639bd99ecb1f7a6faf 2024-11-20T06:41:50,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/36bd78542f8d43e9b89bf87a9b9a18b9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/36bd78542f8d43e9b89bf87a9b9a18b9 2024-11-20T06:41:50,993 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/recovered.edits/304.seqid, newMaxSeqId=304, maxSeqId=4 2024-11-20T06:41:50,994 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. 2024-11-20T06:41:50,994 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 88c2857bcc22b20889f07648453bfe35: Waiting for close lock at 1732084908671Running coprocessor pre-close hooks at 1732084908671Disabling compacts and flushes for region at 1732084908671Disabling writes for close at 1732084909675 (+1004 ms)Obtaining lock to block concurrent updates at 1732084909675Preparing flush snapshotting stores in 88c2857bcc22b20889f07648453bfe35 at 1732084909675Finished memstore snapshotting TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732084909676 (+1 ms)Flushing stores of TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35. at 1732084909676Flushing 88c2857bcc22b20889f07648453bfe35/A: creating writer at 1732084909676Flushing mob file 88c2857bcc22b20889f07648453bfe35/A: appending metadata at 1732084909679 (+3 ms)Flushing mob file 88c2857bcc22b20889f07648453bfe35/A: closing flushed file at 1732084909679Flushing 88c2857bcc22b20889f07648453bfe35/A: appending metadata at 1732084910087 (+408 ms)Flushing 88c2857bcc22b20889f07648453bfe35/A: closing flushed file at 1732084910087Flushing 88c2857bcc22b20889f07648453bfe35/B: creating writer at 1732084910093 (+6 ms)Flushing 88c2857bcc22b20889f07648453bfe35/B: appending metadata at 1732084910095 (+2 ms)Flushing 88c2857bcc22b20889f07648453bfe35/B: closing flushed file at 1732084910095Flushing 88c2857bcc22b20889f07648453bfe35/C: creating writer at 1732084910503 (+408 ms)Flushing 88c2857bcc22b20889f07648453bfe35/C: appending metadata at 1732084910506 (+3 ms)Flushing 88c2857bcc22b20889f07648453bfe35/C: closing flushed file at 1732084910506Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@483af894: reopening flushed file at 1732084910922 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47f6ff77: reopening flushed file at 1732084910927 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4310ed00: reopening flushed file at 1732084910931 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 88c2857bcc22b20889f07648453bfe35 in 1259ms, sequenceid=301, compaction requested=true at 1732084910934 (+3 ms)Writing region close event to WAL at 1732084910989 (+55 ms)Running coprocessor post-close hooks at 1732084910994 (+5 ms)Closed at 1732084910994 2024-11-20T06:41:50,996 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:50,996 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=88c2857bcc22b20889f07648453bfe35, regionState=CLOSED 2024-11-20T06:41:50,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:41:50,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-20T06:41:50,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 88c2857bcc22b20889f07648453bfe35, server=61c2066a8a0f,33847,1732084794617 in 2.4800 sec 2024-11-20T06:41:51,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-20T06:41:51,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88c2857bcc22b20889f07648453bfe35, UNASSIGN in 2.4850 sec 2024-11-20T06:41:51,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-20T06:41:51,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4890 sec 2024-11-20T06:41:51,003 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084911003"}]},"ts":"1732084911003"} 2024-11-20T06:41:51,004 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T06:41:51,004 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T06:41:51,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4980 sec 2024-11-20T06:41:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T06:41:52,675 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:52,676 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T06:41:52,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:52,681 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T06:41:52,682 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:52,687 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,692 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/recovered.edits] 2024-11-20T06:41:52,698 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6126808ce6cc43118b620a14e5c0b8b8 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/6126808ce6cc43118b620a14e5c0b8b8 2024-11-20T06:41:52,701 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/9b03c9dc17d14880b870014a781ca51f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/9b03c9dc17d14880b870014a781ca51f 2024-11-20T06:41:52,704 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/9f3178f150bf4aa783b441f97544c63d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/9f3178f150bf4aa783b441f97544c63d 2024-11-20T06:41:52,706 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/ba23ac27b2a846c587919c88d44e8636 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/ba23ac27b2a846c587919c88d44e8636 2024-11-20T06:41:52,709 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/c1f5e4d49e1a4aa19f0e02eceb491f5c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/A/c1f5e4d49e1a4aa19f0e02eceb491f5c 2024-11-20T06:41:52,714 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/3c1e2cf506774e229b2fb19f1c53910e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/3c1e2cf506774e229b2fb19f1c53910e 2024-11-20T06:41:52,715 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/cae0994d336f4e668bc96d87614ffa88 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/cae0994d336f4e668bc96d87614ffa88 2024-11-20T06:41:52,717 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d333ed1b316e403aabf71b8486c81bb9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d333ed1b316e403aabf71b8486c81bb9 2024-11-20T06:41:52,718 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d41238ba7dfb4eeb93208b00292ad829 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/d41238ba7dfb4eeb93208b00292ad829 2024-11-20T06:41:52,720 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/de1517633fe844e796dd81fa07818405 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/B/de1517633fe844e796dd81fa07818405 2024-11-20T06:41:52,723 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/1825185f09824ffd8d53573cd1fafd60 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/1825185f09824ffd8d53573cd1fafd60 2024-11-20T06:41:52,725 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/a3b5c1f669fc4eeea567135731f1b4e7 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/a3b5c1f669fc4eeea567135731f1b4e7 2024-11-20T06:41:52,728 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/bb06f3d7516e4afb8ecdedfada9c2c44 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/bb06f3d7516e4afb8ecdedfada9c2c44 2024-11-20T06:41:52,730 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/cae67dc8e28847c98a82778b2f2e0ebe to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/cae67dc8e28847c98a82778b2f2e0ebe 2024-11-20T06:41:52,732 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/e71da35248b94c98b02fb1a96faf2969 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/C/e71da35248b94c98b02fb1a96faf2969 2024-11-20T06:41:52,735 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/recovered.edits/304.seqid to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35/recovered.edits/304.seqid 2024-11-20T06:41:52,736 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,736 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T06:41:52,736 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T06:41:52,737 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T06:41:52,739 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200a7fbbe1289f407996a03371e970513b_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200a7fbbe1289f407996a03371e970513b_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,740 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112026697ac030dd463c8a4ce1581fcd625c_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112026697ac030dd463c8a4ce1581fcd625c_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,741 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202bc6cfbe89814cf38296dd0457d5fb1d_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202bc6cfbe89814cf38296dd0457d5fb1d_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,742 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112039a16a299ee54be6895a12a73655a3ba_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112039a16a299ee54be6895a12a73655a3ba_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,743 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203ce56a72f5134d458c8461f863b950a3_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203ce56a72f5134d458c8461f863b950a3_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,744 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204ebf0ae346f94e098acf1b00cbed1c64_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204ebf0ae346f94e098acf1b00cbed1c64_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,745 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120745352bc36b1421bbb1564de048ef564_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120745352bc36b1421bbb1564de048ef564_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,746 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120927f13cf798040a5bd9ad5ecda9209e8_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120927f13cf798040a5bd9ad5ecda9209e8_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,747 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209f40d69b32ff4d6a82c29c93655ff4c1_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209f40d69b32ff4d6a82c29c93655ff4c1_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,748 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a6ccf02ff8e4420a959a717e80981e04_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a6ccf02ff8e4420a959a717e80981e04_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,749 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120adba2c65bc584cb2a6e8e893d3473f9c_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120adba2c65bc584cb2a6e8e893d3473f9c_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,750 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b98c623f8acf45ec89084690a2158253_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b98c623f8acf45ec89084690a2158253_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,751 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120baef0ab2393e4e9fa2697ae5f688921c_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120baef0ab2393e4e9fa2697ae5f688921c_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,752 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dea606fb7d074c1eb9bc0b5543062502_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dea606fb7d074c1eb9bc0b5543062502_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,753 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f88399ab5e4b49c189ad1e93743e8296_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f88399ab5e4b49c189ad1e93743e8296_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,754 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f8ac24780c8b4f5c96add619d9b0af04_88c2857bcc22b20889f07648453bfe35 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f8ac24780c8b4f5c96add619d9b0af04_88c2857bcc22b20889f07648453bfe35 2024-11-20T06:41:52,754 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T06:41:52,756 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:52,757 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T06:41:52,759 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T06:41:52,760 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:52,760 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T06:41:52,760 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732084912760"}]},"ts":"9223372036854775807"} 2024-11-20T06:41:52,761 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T06:41:52,761 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 88c2857bcc22b20889f07648453bfe35, NAME => 'TestAcidGuarantees,,1732084884306.88c2857bcc22b20889f07648453bfe35.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T06:41:52,762 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T06:41:52,762 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732084912762"}]},"ts":"9223372036854775807"} 2024-11-20T06:41:52,763 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T06:41:52,764 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:52,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 88 msec 2024-11-20T06:41:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T06:41:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T06:41:52,797 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:52,806 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 238) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1351162969_22 at /127.0.0.1:37812 [Waiting for operation #340] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1532015227_22 at /127.0.0.1:37904 [Waiting for operation #298] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1351162969_22 at /127.0.0.1:37906 [Waiting for operation #294] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=404 (was 446), ProcessCount=11 (was 11), AvailableMemoryMB=4439 (was 4601) 2024-11-20T06:41:52,817 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=404, ProcessCount=11, AvailableMemoryMB=4439 2024-11-20T06:41:52,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T06:41:52,819 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:41:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T06:41:52,821 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T06:41:52,821 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:52,821 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-20T06:41:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-20T06:41:52,822 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T06:41:52,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742254_1430 (size=959) 2024-11-20T06:41:52,878 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T06:41:52,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-20T06:41:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-20T06:41:53,234 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5 2024-11-20T06:41:53,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742255_1431 (size=53) 2024-11-20T06:41:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-20T06:41:53,642 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:41:53,642 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6bac99491880cd23064dd43f1153d648, disabling compactions & flushes 2024-11-20T06:41:53,642 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:53,643 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:53,643 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. after waiting 0 ms 2024-11-20T06:41:53,643 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:53,643 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:53,643 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6bac99491880cd23064dd43f1153d648: Waiting for close lock at 1732084913642Disabling compacts and flushes for region at 1732084913642Disabling writes for close at 1732084913643 (+1 ms)Writing region close event to WAL at 1732084913643Closed at 1732084913643 2024-11-20T06:41:53,644 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T06:41:53,645 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732084913644"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732084913644"}]},"ts":"1732084913644"} 2024-11-20T06:41:53,647 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T06:41:53,648 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T06:41:53,648 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084913648"}]},"ts":"1732084913648"} 2024-11-20T06:41:53,650 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T06:41:53,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bac99491880cd23064dd43f1153d648, ASSIGN}] 2024-11-20T06:41:53,652 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bac99491880cd23064dd43f1153d648, ASSIGN 2024-11-20T06:41:53,653 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bac99491880cd23064dd43f1153d648, ASSIGN; state=OFFLINE, location=61c2066a8a0f,33847,1732084794617; forceNewPlan=false, retain=false 2024-11-20T06:41:53,804 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=6bac99491880cd23064dd43f1153d648, regionState=OPENING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:53,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bac99491880cd23064dd43f1153d648, ASSIGN because future has completed 2024-11-20T06:41:53,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:41:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-20T06:41:53,968 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:53,968 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:41:53,969 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,969 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:41:53,969 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,969 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,970 INFO [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,972 INFO [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bac99491880cd23064dd43f1153d648 columnFamilyName A 2024-11-20T06:41:53,972 DEBUG [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:53,973 INFO [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] regionserver.HStore(327): Store=6bac99491880cd23064dd43f1153d648/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:41:53,973 INFO [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,974 INFO [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bac99491880cd23064dd43f1153d648 columnFamilyName B 2024-11-20T06:41:53,974 DEBUG [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:53,975 INFO [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] regionserver.HStore(327): Store=6bac99491880cd23064dd43f1153d648/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:41:53,975 INFO [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,976 INFO [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bac99491880cd23064dd43f1153d648 columnFamilyName C 2024-11-20T06:41:53,976 DEBUG [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:41:53,977 INFO [StoreOpener-6bac99491880cd23064dd43f1153d648-1 {}] regionserver.HStore(327): Store=6bac99491880cd23064dd43f1153d648/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:41:53,977 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,978 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,979 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,980 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,980 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,981 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:41:53,983 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,985 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:41:53,985 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 6bac99491880cd23064dd43f1153d648; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65236022, jitterRate=-0.027907520532608032}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:41:53,985 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:53,986 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 6bac99491880cd23064dd43f1153d648: Running coprocessor pre-open hook at 1732084913969Writing region info on filesystem at 1732084913969Initializing all the Stores at 1732084913970 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084913970Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084913970Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084913970Cleaning up temporary data from old regions at 1732084913980 (+10 ms)Running coprocessor post-open hooks at 1732084913985 (+5 ms)Region opened successfully at 1732084913986 (+1 ms) 2024-11-20T06:41:53,986 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., pid=120, masterSystemTime=1732084913964 2024-11-20T06:41:53,988 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:53,988 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:53,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=6bac99491880cd23064dd43f1153d648, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:53,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:41:53,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-20T06:41:53,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 in 181 msec 2024-11-20T06:41:53,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-20T06:41:53,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bac99491880cd23064dd43f1153d648, ASSIGN in 340 msec 2024-11-20T06:41:53,993 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T06:41:53,993 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084913993"}]},"ts":"1732084913993"} 2024-11-20T06:41:53,994 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T06:41:53,995 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T06:41:53,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-20T06:41:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-20T06:41:54,986 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:54,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47e1a424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:54,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:54,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:54,992 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:54,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5a3e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:54,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:54,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:54,995 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e15b00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:54,996 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:54,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:54,999 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T06:41:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,33659,1732084793752 2024-11-20T06:41:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f7fdf9f 2024-11-20T06:41:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T06:41:55,001 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T06:41:55,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407ebc73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,003 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:55,003 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:55,004 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:55,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:55,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:55,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60312ad2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:55,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:55,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:55,005 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:55,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23a4ca57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5159e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:55,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:55,007 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a6837e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:55,009 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50bb9f06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1062bfdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:55,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:55,011 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3169f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:55,012 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279e353a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7232e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:55,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:55,014 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@773111ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:55,016 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d0b5040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39880b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:55,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:55,019 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:55,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:55,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5360edc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:55,021 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b547e3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb46e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:55,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:55,023 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7dde46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:55,025 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a8c357c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1593fd05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:55,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:55,028 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e77ceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:55,029 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c757797, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@729114ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:55,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:55,031 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19fab789, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:55,032 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b0812d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f4c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:55,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:55,034 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d451f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:55,035 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23634693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b03bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:41:55,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:41:55,037 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:41:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:41:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:41:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f229a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:41:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:41:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:41:55,038 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:41:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147aaf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:41:55,039 DEBUG [Thread-1937 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:55,039 DEBUG [Thread-1956 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:55,039 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:55,040 DEBUG [Thread-1941 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-20T06:41:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:55,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T06:41:55,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:55,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:55,042 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:55,042 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:41:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:41:55,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:41:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:55,049 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:55,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T06:41:55,055 DEBUG [Thread-1948 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:55,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:55,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:55,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:55,059 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:55,059 DEBUG [Thread-1950 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:41:55,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:55,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:55,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:55,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:55,063 DEBUG [Thread-1954 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:41:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:41:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:55,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:41:55,069 DEBUG [Thread-1939 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:55,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:55,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/8bf3239302784d01b36f8e071e3af7f9 is 50, key is test_row_0/A:col10/1732084915048/Put/seqid=0 2024-11-20T06:41:55,072 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:41:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:55,075 DEBUG [Thread-1952 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:41:55,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:41:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:41:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:41:55,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:41:55,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742256_1432 (size=12001) 2024-11-20T06:41:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2] 2024-11-20T06:41:55,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/8bf3239302784d01b36f8e071e3af7f9 2024-11-20T06:41:55,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084975070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084975074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084975078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084975077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084975080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/df90500067504e309d77248850b4a9f3 is 50, key is test_row_0/B:col10/1732084915048/Put/seqid=0 2024-11-20T06:41:55,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742257_1433 (size=12001) 2024-11-20T06:41:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T06:41:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084975183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084975183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084975183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084975184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T06:41:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:55,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084975194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T06:41:55,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:55,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T06:41:55,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084975393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084975394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084975394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084975394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084975404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T06:41:55,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/df90500067504e309d77248850b4a9f3 2024-11-20T06:41:55,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/d03c0d4f7ee84192920a26a591c4e5c0 is 50, key is test_row_0/C:col10/1732084915048/Put/seqid=0 2024-11-20T06:41:55,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742258_1434 (size=12001) 2024-11-20T06:41:55,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T06:41:55,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:55,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T06:41:55,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084975703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084975703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084975703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084975703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084975714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:55,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:55,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T06:41:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:55,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/d03c0d4f7ee84192920a26a591c4e5c0 2024-11-20T06:41:55,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/8bf3239302784d01b36f8e071e3af7f9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/8bf3239302784d01b36f8e071e3af7f9 2024-11-20T06:41:55,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/8bf3239302784d01b36f8e071e3af7f9, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T06:41:55,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/df90500067504e309d77248850b4a9f3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/df90500067504e309d77248850b4a9f3 2024-11-20T06:41:55,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/df90500067504e309d77248850b4a9f3, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T06:41:55,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/d03c0d4f7ee84192920a26a591c4e5c0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/d03c0d4f7ee84192920a26a591c4e5c0 2024-11-20T06:41:55,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/d03c0d4f7ee84192920a26a591c4e5c0, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T06:41:55,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6bac99491880cd23064dd43f1153d648 in 896ms, sequenceid=13, compaction requested=false 2024-11-20T06:41:55,945 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T06:41:55,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:41:55,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T06:41:55,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:55,956 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T06:41:55,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/d318ddaf4fd74e54917434a07af76dce is 50, key is test_row_0/A:col10/1732084915078/Put/seqid=0 2024-11-20T06:41:55,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742259_1435 (size=12001) 2024-11-20T06:41:55,965 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/d318ddaf4fd74e54917434a07af76dce 2024-11-20T06:41:55,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/c460d488549746bb8c9436667c8b84d9 is 50, key is test_row_0/B:col10/1732084915078/Put/seqid=0 2024-11-20T06:41:55,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742260_1436 (size=12001) 2024-11-20T06:41:55,974 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/c460d488549746bb8c9436667c8b84d9 2024-11-20T06:41:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/34cbc8cc10964cb6b39fbcb1a3b82960 is 50, key is test_row_0/C:col10/1732084915078/Put/seqid=0 2024-11-20T06:41:55,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742261_1437 (size=12001) 2024-11-20T06:41:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T06:41:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:56,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:56,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084976225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084976225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084976226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084976228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084976229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084976334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084976334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084976343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084976343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084976343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,384 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/34cbc8cc10964cb6b39fbcb1a3b82960 2024-11-20T06:41:56,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/d318ddaf4fd74e54917434a07af76dce as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/d318ddaf4fd74e54917434a07af76dce 2024-11-20T06:41:56,390 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/d318ddaf4fd74e54917434a07af76dce, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T06:41:56,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/c460d488549746bb8c9436667c8b84d9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/c460d488549746bb8c9436667c8b84d9 2024-11-20T06:41:56,395 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/c460d488549746bb8c9436667c8b84d9, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T06:41:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/34cbc8cc10964cb6b39fbcb1a3b82960 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/34cbc8cc10964cb6b39fbcb1a3b82960 2024-11-20T06:41:56,398 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/34cbc8cc10964cb6b39fbcb1a3b82960, entries=150, sequenceid=37, filesize=11.7 K 2024-11-20T06:41:56,399 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6bac99491880cd23064dd43f1153d648 in 443ms, sequenceid=37, compaction requested=false 2024-11-20T06:41:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:41:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-20T06:41:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-20T06:41:56,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-20T06:41:56,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3580 sec 2024-11-20T06:41:56,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.3630 sec 2024-11-20T06:41:56,516 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T06:41:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:56,546 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:56,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T06:41:56,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/f2a234bb4f3a4e4f8a0b8b3bde7ebd33 is 50, key is test_row_0/A:col10/1732084916545/Put/seqid=0 2024-11-20T06:41:56,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742262_1438 (size=14341) 2024-11-20T06:41:56,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/f2a234bb4f3a4e4f8a0b8b3bde7ebd33 2024-11-20T06:41:56,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/fbf6331b92f84fb89baba29d029b55f3 is 50, key is test_row_0/B:col10/1732084916545/Put/seqid=0 2024-11-20T06:41:56,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742263_1439 (size=12001) 2024-11-20T06:41:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084976590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084976621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084976621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084976621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084976622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084976733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084976743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084976744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084976744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084976744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084976943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084976953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084976954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084976954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084976954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:56,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/fbf6331b92f84fb89baba29d029b55f3 2024-11-20T06:41:56,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/247750ce38bd4f5f8a3fbf06588e6ab5 is 50, key is test_row_0/C:col10/1732084916545/Put/seqid=0 2024-11-20T06:41:56,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742264_1440 (size=12001) 2024-11-20T06:41:56,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/247750ce38bd4f5f8a3fbf06588e6ab5 2024-11-20T06:41:56,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/f2a234bb4f3a4e4f8a0b8b3bde7ebd33 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f2a234bb4f3a4e4f8a0b8b3bde7ebd33 2024-11-20T06:41:56,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f2a234bb4f3a4e4f8a0b8b3bde7ebd33, entries=200, sequenceid=50, filesize=14.0 K 2024-11-20T06:41:56,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/fbf6331b92f84fb89baba29d029b55f3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/fbf6331b92f84fb89baba29d029b55f3 2024-11-20T06:41:56,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/fbf6331b92f84fb89baba29d029b55f3, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T06:41:56,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/247750ce38bd4f5f8a3fbf06588e6ab5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/247750ce38bd4f5f8a3fbf06588e6ab5 2024-11-20T06:41:56,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/247750ce38bd4f5f8a3fbf06588e6ab5, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T06:41:56,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6bac99491880cd23064dd43f1153d648 in 450ms, sequenceid=50, compaction requested=true 2024-11-20T06:41:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:41:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:41:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:56,996 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:41:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:56,996 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:41:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:56,997 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:56,997 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:56,997 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/B is initiating minor compaction (all files) 2024-11-20T06:41:56,997 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/A is initiating minor compaction (all files) 2024-11-20T06:41:56,997 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/B in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:56,997 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/A in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:56,997 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/8bf3239302784d01b36f8e071e3af7f9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/d318ddaf4fd74e54917434a07af76dce, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f2a234bb4f3a4e4f8a0b8b3bde7ebd33] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=37.4 K 2024-11-20T06:41:56,997 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/df90500067504e309d77248850b4a9f3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/c460d488549746bb8c9436667c8b84d9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/fbf6331b92f84fb89baba29d029b55f3] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=35.2 K 2024-11-20T06:41:56,997 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting df90500067504e309d77248850b4a9f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732084915047 2024-11-20T06:41:56,997 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bf3239302784d01b36f8e071e3af7f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732084915047 2024-11-20T06:41:56,997 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c460d488549746bb8c9436667c8b84d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732084915068 2024-11-20T06:41:56,998 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting d318ddaf4fd74e54917434a07af76dce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732084915068 2024-11-20T06:41:56,998 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf6331b92f84fb89baba29d029b55f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084916227 2024-11-20T06:41:56,998 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2a234bb4f3a4e4f8a0b8b3bde7ebd33, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084916220 2024-11-20T06:41:57,003 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#A#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:57,003 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:57,004 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/42dd3fcbeb0f4beca7d7bc76991f3107 is 50, key is test_row_0/A:col10/1732084916545/Put/seqid=0 2024-11-20T06:41:57,004 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/a7c01e5bbe8c4576a2f47c0265d172bc is 50, key is test_row_0/B:col10/1732084916545/Put/seqid=0 2024-11-20T06:41:57,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742265_1441 (size=12104) 2024-11-20T06:41:57,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742266_1442 (size=12104) 2024-11-20T06:41:57,020 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/a7c01e5bbe8c4576a2f47c0265d172bc as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a7c01e5bbe8c4576a2f47c0265d172bc 2024-11-20T06:41:57,020 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/42dd3fcbeb0f4beca7d7bc76991f3107 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/42dd3fcbeb0f4beca7d7bc76991f3107 2024-11-20T06:41:57,025 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/A of 6bac99491880cd23064dd43f1153d648 into 42dd3fcbeb0f4beca7d7bc76991f3107(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:57,025 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:41:57,025 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/A, priority=13, startTime=1732084916996; duration=0sec 2024-11-20T06:41:57,025 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:41:57,025 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:A 2024-11-20T06:41:57,025 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:41:57,026 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:41:57,027 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/C is initiating minor compaction (all files) 2024-11-20T06:41:57,027 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/C in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,027 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/d03c0d4f7ee84192920a26a591c4e5c0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/34cbc8cc10964cb6b39fbcb1a3b82960, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/247750ce38bd4f5f8a3fbf06588e6ab5] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=35.2 K 2024-11-20T06:41:57,027 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting d03c0d4f7ee84192920a26a591c4e5c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732084915047 2024-11-20T06:41:57,027 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/B of 6bac99491880cd23064dd43f1153d648 into a7c01e5bbe8c4576a2f47c0265d172bc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:57,027 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:41:57,027 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/B, priority=13, startTime=1732084916996; duration=0sec 2024-11-20T06:41:57,027 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34cbc8cc10964cb6b39fbcb1a3b82960, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732084915068 2024-11-20T06:41:57,027 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:57,027 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:B 2024-11-20T06:41:57,030 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 247750ce38bd4f5f8a3fbf06588e6ab5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084916227 2024-11-20T06:41:57,037 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#C#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:41:57,037 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/30d98c939e21425788a8a897cfe64da4 is 50, key is test_row_0/C:col10/1732084916545/Put/seqid=0 2024-11-20T06:41:57,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742267_1443 (size=12104) 2024-11-20T06:41:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T06:41:57,194 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:41:57,194 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:41:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-20T06:41:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T06:41:57,196 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:41:57,197 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:41:57,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:41:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:57,259 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:41:57,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-20T06:41:57,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/e40f36a73eb640f09a47d999ad6d76e3 is 50, key is test_row_0/A:col10/1732084916593/Put/seqid=0 2024-11-20T06:41:57,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742268_1444 (size=14341) 2024-11-20T06:41:57,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084977274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084977274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084977276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084977280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084977281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T06:41:57,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T06:41:57,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:57,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084977384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084977384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084977393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084977393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,451 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/30d98c939e21425788a8a897cfe64da4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/30d98c939e21425788a8a897cfe64da4 2024-11-20T06:41:57,459 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/C of 6bac99491880cd23064dd43f1153d648 into 30d98c939e21425788a8a897cfe64da4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:41:57,459 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:41:57,459 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/C, priority=13, startTime=1732084916996; duration=0sec 2024-11-20T06:41:57,459 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:41:57,459 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:C 2024-11-20T06:41:57,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T06:41:57,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:57,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T06:41:57,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084977594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084977594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084977603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084977603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T06:41:57,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:57,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/e40f36a73eb640f09a47d999ad6d76e3 2024-11-20T06:41:57,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/6079a71c390e4b1dba991c333e3ffb1d is 50, key is test_row_0/B:col10/1732084916593/Put/seqid=0 2024-11-20T06:41:57,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742269_1445 (size=12001) 2024-11-20T06:41:57,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/8bf3239302784d01b36f8e071e3af7f9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/d318ddaf4fd74e54917434a07af76dce, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f2a234bb4f3a4e4f8a0b8b3bde7ebd33] to archive 2024-11-20T06:41:57,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:41:57,767 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/8bf3239302784d01b36f8e071e3af7f9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/8bf3239302784d01b36f8e071e3af7f9 2024-11-20T06:41:57,768 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/d318ddaf4fd74e54917434a07af76dce to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/d318ddaf4fd74e54917434a07af76dce 2024-11-20T06:41:57,769 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f2a234bb4f3a4e4f8a0b8b3bde7ebd33 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f2a234bb4f3a4e4f8a0b8b3bde7ebd33 2024-11-20T06:41:57,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/df90500067504e309d77248850b4a9f3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/c460d488549746bb8c9436667c8b84d9, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/fbf6331b92f84fb89baba29d029b55f3] to archive 2024-11-20T06:41:57,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:41:57,771 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/df90500067504e309d77248850b4a9f3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/df90500067504e309d77248850b4a9f3 2024-11-20T06:41:57,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/c460d488549746bb8c9436667c8b84d9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/c460d488549746bb8c9436667c8b84d9 2024-11-20T06:41:57,773 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/fbf6331b92f84fb89baba29d029b55f3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/fbf6331b92f84fb89baba29d029b55f3 2024-11-20T06:41:57,774 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/d03c0d4f7ee84192920a26a591c4e5c0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/34cbc8cc10964cb6b39fbcb1a3b82960, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/247750ce38bd4f5f8a3fbf06588e6ab5] to archive 2024-11-20T06:41:57,774 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:41:57,775 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/d03c0d4f7ee84192920a26a591c4e5c0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/d03c0d4f7ee84192920a26a591c4e5c0 2024-11-20T06:41:57,776 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/34cbc8cc10964cb6b39fbcb1a3b82960 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/34cbc8cc10964cb6b39fbcb1a3b82960 2024-11-20T06:41:57,777 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/247750ce38bd4f5f8a3fbf06588e6ab5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/247750ce38bd4f5f8a3fbf06588e6ab5 2024-11-20T06:41:57,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084977794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T06:41:57,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:57,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T06:41:57,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084977904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084977904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084977914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084977914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:57,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T06:41:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:57,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:58,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/6079a71c390e4b1dba991c333e3ffb1d 2024-11-20T06:41:58,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/6c60e12ba7ce4e7787a88576ce75536d is 50, key is test_row_0/C:col10/1732084916593/Put/seqid=0 2024-11-20T06:41:58,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742270_1446 (size=12001) 2024-11-20T06:41:58,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T06:41:58,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:58,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:58,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:58,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:58,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:58,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T06:41:58,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:58,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:58,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:58,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:58,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T06:41:58,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T06:41:58,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:58,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:58,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:58,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:58,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:41:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084978414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084978414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:58,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084978424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:58,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084978424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:58,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/6c60e12ba7ce4e7787a88576ce75536d 2024-11-20T06:41:58,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/e40f36a73eb640f09a47d999ad6d76e3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e40f36a73eb640f09a47d999ad6d76e3 2024-11-20T06:41:58,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e40f36a73eb640f09a47d999ad6d76e3, entries=200, sequenceid=76, filesize=14.0 K 2024-11-20T06:41:58,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/6079a71c390e4b1dba991c333e3ffb1d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/6079a71c390e4b1dba991c333e3ffb1d 2024-11-20T06:41:58,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/6079a71c390e4b1dba991c333e3ffb1d, entries=150, sequenceid=76, filesize=11.7 K 2024-11-20T06:41:58,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/6c60e12ba7ce4e7787a88576ce75536d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6c60e12ba7ce4e7787a88576ce75536d 2024-11-20T06:41:58,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6c60e12ba7ce4e7787a88576ce75536d, entries=150, sequenceid=76, filesize=11.7 K 2024-11-20T06:41:58,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6bac99491880cd23064dd43f1153d648 in 1242ms, sequenceid=76, compaction requested=false 2024-11-20T06:41:58,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:41:58,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T06:41:58,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:58,570 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T06:41:58,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/edc3e64ea5ec44e89f5c1da6e7eef52e is 50, key is test_row_0/A:col10/1732084917275/Put/seqid=0 2024-11-20T06:41:58,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742271_1447 (size=12001) 2024-11-20T06:41:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:41:58,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:41:58,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084978910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:58,978 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/edc3e64ea5ec44e89f5c1da6e7eef52e 2024-11-20T06:41:58,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/0ce059e1d97346b3a9efb8b10cd072bd is 50, key is test_row_0/B:col10/1732084917275/Put/seqid=0 2024-11-20T06:41:58,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742272_1448 (size=12001) 2024-11-20T06:41:59,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084979024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:59,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084979233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T06:41:59,393 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/0ce059e1d97346b3a9efb8b10cd072bd 2024-11-20T06:41:59,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/240d361f9b794fbcba77089358269c6b is 50, key is test_row_0/C:col10/1732084917275/Put/seqid=0 2024-11-20T06:41:59,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742273_1449 (size=12001) 2024-11-20T06:41:59,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084979434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:59,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084979434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:59,440 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in 6bac99491880cd23064dd43f1153d648 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:41:59,440 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in 6bac99491880cd23064dd43f1153d648 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:41:59,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:59,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:59,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084979443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:59,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084979443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:59,449 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in 6bac99491880cd23064dd43f1153d648 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:41:59,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,449 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in 6bac99491880cd23064dd43f1153d648 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:41:59,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:59,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:59,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:41:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084979543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:41:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:41:59,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:41:59,801 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/240d361f9b794fbcba77089358269c6b 2024-11-20T06:41:59,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/edc3e64ea5ec44e89f5c1da6e7eef52e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/edc3e64ea5ec44e89f5c1da6e7eef52e 2024-11-20T06:41:59,807 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/edc3e64ea5ec44e89f5c1da6e7eef52e, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T06:41:59,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/0ce059e1d97346b3a9efb8b10cd072bd as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/0ce059e1d97346b3a9efb8b10cd072bd 2024-11-20T06:41:59,811 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/0ce059e1d97346b3a9efb8b10cd072bd, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T06:41:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/240d361f9b794fbcba77089358269c6b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/240d361f9b794fbcba77089358269c6b 2024-11-20T06:41:59,814 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/240d361f9b794fbcba77089358269c6b, entries=150, sequenceid=89, filesize=11.7 K 2024-11-20T06:41:59,815 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6bac99491880cd23064dd43f1153d648 in 1245ms, sequenceid=89, compaction requested=true 2024-11-20T06:41:59,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:41:59,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:41:59,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-20T06:41:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-20T06:41:59,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-20T06:41:59,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6190 sec 2024-11-20T06:41:59,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.6250 sec 2024-11-20T06:42:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:00,059 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:00,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T06:42:00,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/6935f84549104ae5a5bb9ea2e37164e6 is 50, key is test_row_0/A:col10/1732084920058/Put/seqid=0 2024-11-20T06:42:00,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742274_1450 (size=14341) 2024-11-20T06:42:00,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/6935f84549104ae5a5bb9ea2e37164e6 2024-11-20T06:42:00,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/5f3b06bc562d4aba8b92792ab94d4232 is 50, key is test_row_0/B:col10/1732084920058/Put/seqid=0 2024-11-20T06:42:00,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742275_1451 (size=12001) 2024-11-20T06:42:00,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/5f3b06bc562d4aba8b92792ab94d4232 2024-11-20T06:42:00,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/98a6fe31e0ed403ba427125ca77a2447 is 50, key is test_row_0/C:col10/1732084920058/Put/seqid=0 2024-11-20T06:42:00,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742276_1452 (size=12001) 2024-11-20T06:42:00,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084980124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:00,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084980243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:00,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:00,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:00,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:00,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084980454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:00,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/98a6fe31e0ed403ba427125ca77a2447 2024-11-20T06:42:00,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/6935f84549104ae5a5bb9ea2e37164e6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6935f84549104ae5a5bb9ea2e37164e6 2024-11-20T06:42:00,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6935f84549104ae5a5bb9ea2e37164e6, entries=200, sequenceid=114, filesize=14.0 K 2024-11-20T06:42:00,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/5f3b06bc562d4aba8b92792ab94d4232 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5f3b06bc562d4aba8b92792ab94d4232 2024-11-20T06:42:00,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5f3b06bc562d4aba8b92792ab94d4232, entries=150, sequenceid=114, filesize=11.7 K 2024-11-20T06:42:00,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/98a6fe31e0ed403ba427125ca77a2447 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/98a6fe31e0ed403ba427125ca77a2447 2024-11-20T06:42:00,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/98a6fe31e0ed403ba427125ca77a2447, entries=150, sequenceid=114, filesize=11.7 K 2024-11-20T06:42:00,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6bac99491880cd23064dd43f1153d648 in 441ms, sequenceid=114, compaction requested=true 2024-11-20T06:42:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:00,500 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:42:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T06:42:00,500 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:42:00,502 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:42:00,502 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/B is initiating minor compaction (all files) 2024-11-20T06:42:00,502 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/B in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:00,502 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a7c01e5bbe8c4576a2f47c0265d172bc, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/6079a71c390e4b1dba991c333e3ffb1d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/0ce059e1d97346b3a9efb8b10cd072bd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5f3b06bc562d4aba8b92792ab94d4232] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=47.0 K 2024-11-20T06:42:00,502 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:42:00,502 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/A is initiating minor compaction (all files) 2024-11-20T06:42:00,502 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/A in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:00,502 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c01e5bbe8c4576a2f47c0265d172bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084916227 2024-11-20T06:42:00,502 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/42dd3fcbeb0f4beca7d7bc76991f3107, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e40f36a73eb640f09a47d999ad6d76e3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/edc3e64ea5ec44e89f5c1da6e7eef52e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6935f84549104ae5a5bb9ea2e37164e6] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=51.5 K 2024-11-20T06:42:00,503 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 6079a71c390e4b1dba991c333e3ffb1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732084916588 2024-11-20T06:42:00,503 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42dd3fcbeb0f4beca7d7bc76991f3107, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084916227 2024-11-20T06:42:00,503 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ce059e1d97346b3a9efb8b10cd072bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732084917269 2024-11-20T06:42:00,503 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting e40f36a73eb640f09a47d999ad6d76e3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732084916588 2024-11-20T06:42:00,503 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f3b06bc562d4aba8b92792ab94d4232, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732084918901 2024-11-20T06:42:00,503 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting edc3e64ea5ec44e89f5c1da6e7eef52e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732084917269 2024-11-20T06:42:00,503 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6935f84549104ae5a5bb9ea2e37164e6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732084918899 2024-11-20T06:42:00,518 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#B#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:00,518 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#A#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:00,518 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/02c07510238448ea88e86efe664539b2 is 50, key is test_row_0/B:col10/1732084920058/Put/seqid=0 2024-11-20T06:42:00,518 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/30ad491ee28240a5b913e8c3a1c1fabd is 50, key is test_row_0/A:col10/1732084920058/Put/seqid=0 2024-11-20T06:42:00,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742277_1453 (size=12139) 2024-11-20T06:42:00,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742278_1454 (size=12139) 2024-11-20T06:42:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:00,768 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:00,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-20T06:42:00,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/f4742057272245f3ab7aa4f1ed0aa8a2 is 50, key is test_row_0/A:col10/1732084920767/Put/seqid=0 2024-11-20T06:42:00,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742279_1455 (size=14341) 2024-11-20T06:42:00,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084980904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:00,931 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/02c07510238448ea88e86efe664539b2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/02c07510238448ea88e86efe664539b2 2024-11-20T06:42:00,931 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/30ad491ee28240a5b913e8c3a1c1fabd as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/30ad491ee28240a5b913e8c3a1c1fabd 2024-11-20T06:42:00,934 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bac99491880cd23064dd43f1153d648/B of 6bac99491880cd23064dd43f1153d648 into 02c07510238448ea88e86efe664539b2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:00,934 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:00,934 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/B, priority=12, startTime=1732084920500; duration=0sec 2024-11-20T06:42:00,934 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bac99491880cd23064dd43f1153d648/A of 6bac99491880cd23064dd43f1153d648 into 30ad491ee28240a5b913e8c3a1c1fabd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:00,934 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:00,934 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/A, priority=12, startTime=1732084920500; duration=0sec 2024-11-20T06:42:00,934 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:00,934 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:00,934 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:B 2024-11-20T06:42:00,934 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:A 2024-11-20T06:42:00,934 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:42:00,935 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:42:00,935 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/C is initiating minor compaction (all files) 2024-11-20T06:42:00,935 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/C in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:00,935 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/30d98c939e21425788a8a897cfe64da4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6c60e12ba7ce4e7787a88576ce75536d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/240d361f9b794fbcba77089358269c6b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/98a6fe31e0ed403ba427125ca77a2447] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=47.0 K 2024-11-20T06:42:00,935 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30d98c939e21425788a8a897cfe64da4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732084916227 2024-11-20T06:42:00,936 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c60e12ba7ce4e7787a88576ce75536d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732084916588 2024-11-20T06:42:00,936 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 240d361f9b794fbcba77089358269c6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732084917269 2024-11-20T06:42:00,936 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98a6fe31e0ed403ba427125ca77a2447, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732084918901 2024-11-20T06:42:00,951 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#C#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:00,951 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/221b8a1c1b204400ae00ecebf2352c57 is 50, key is test_row_0/C:col10/1732084920058/Put/seqid=0 2024-11-20T06:42:00,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742280_1456 (size=12139) 2024-11-20T06:42:01,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084981014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:01,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/f4742057272245f3ab7aa4f1ed0aa8a2 2024-11-20T06:42:01,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/a65288cdb0674ffc8181394adc719226 is 50, key is test_row_0/B:col10/1732084920767/Put/seqid=0 2024-11-20T06:42:01,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742281_1457 (size=12001) 2024-11-20T06:42:01,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084981223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:01,358 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/221b8a1c1b204400ae00ecebf2352c57 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/221b8a1c1b204400ae00ecebf2352c57 2024-11-20T06:42:01,361 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6bac99491880cd23064dd43f1153d648/C of 6bac99491880cd23064dd43f1153d648 into 221b8a1c1b204400ae00ecebf2352c57(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:01,361 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:01,361 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/C, priority=12, startTime=1732084920500; duration=0sec 2024-11-20T06:42:01,362 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:01,362 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:C 2024-11-20T06:42:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T06:42:01,364 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:01,364 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-20T06:42:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T06:42:01,366 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:01,367 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:01,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:01,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084981453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:01,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084981453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:01,459 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in 6bac99491880cd23064dd43f1153d648 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:01,460 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in 6bac99491880cd23064dd43f1153d648 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:01,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084981464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:01,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084981464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:01,468 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in 6bac99491880cd23064dd43f1153d648 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:01,468 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in 6bac99491880cd23064dd43f1153d648 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T06:42:01,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T06:42:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:01,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084981544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:01,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/a65288cdb0674ffc8181394adc719226 2024-11-20T06:42:01,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/a8e394010d9e4560b0b36884d484985f is 50, key is test_row_0/C:col10/1732084920767/Put/seqid=0 2024-11-20T06:42:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742282_1458 (size=12001) 2024-11-20T06:42:01,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T06:42:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:01,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T06:42:01,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T06:42:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:01,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T06:42:01,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:01,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:01,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:01,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/a8e394010d9e4560b0b36884d484985f 2024-11-20T06:42:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T06:42:01,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/f4742057272245f3ab7aa4f1ed0aa8a2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f4742057272245f3ab7aa4f1ed0aa8a2 2024-11-20T06:42:01,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f4742057272245f3ab7aa4f1ed0aa8a2, entries=200, sequenceid=126, filesize=14.0 K 2024-11-20T06:42:02,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/a65288cdb0674ffc8181394adc719226 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a65288cdb0674ffc8181394adc719226 2024-11-20T06:42:02,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a65288cdb0674ffc8181394adc719226, entries=150, sequenceid=126, filesize=11.7 K 2024-11-20T06:42:02,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/a8e394010d9e4560b0b36884d484985f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/a8e394010d9e4560b0b36884d484985f 2024-11-20T06:42:02,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/a8e394010d9e4560b0b36884d484985f, entries=150, sequenceid=126, filesize=11.7 K 2024-11-20T06:42:02,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6bac99491880cd23064dd43f1153d648 in 1239ms, sequenceid=126, compaction requested=false 2024-11-20T06:42:02,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:02,057 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:02,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T06:42:02,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/15adc96db91f452c9bc5da8240762326 is 50, key is test_row_0/A:col10/1732084920890/Put/seqid=0 2024-11-20T06:42:02,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742283_1459 (size=14541) 2024-11-20T06:42:02,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084982106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:02,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:02,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:02,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:02,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T06:42:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084982214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:02,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T06:42:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084982423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:02,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T06:42:02,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:02,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/15adc96db91f452c9bc5da8240762326 2024-11-20T06:42:02,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/f71ae55e30ff42d6997b1de88044b1de is 50, key is test_row_0/B:col10/1732084920890/Put/seqid=0 2024-11-20T06:42:02,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742284_1460 (size=12151) 2024-11-20T06:42:02,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/f71ae55e30ff42d6997b1de88044b1de 2024-11-20T06:42:02,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/1a6a46a815b34a59a80d7d88a702ef21 is 50, key is test_row_0/C:col10/1732084920890/Put/seqid=0 2024-11-20T06:42:02,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742285_1461 (size=12151) 2024-11-20T06:42:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T06:42:02,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T06:42:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084982744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:02,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:02,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:02,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:02,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T06:42:02,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:02,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:02,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/1a6a46a815b34a59a80d7d88a702ef21 2024-11-20T06:42:02,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/15adc96db91f452c9bc5da8240762326 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/15adc96db91f452c9bc5da8240762326 2024-11-20T06:42:02,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/15adc96db91f452c9bc5da8240762326, entries=200, sequenceid=154, filesize=14.2 K 2024-11-20T06:42:02,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/f71ae55e30ff42d6997b1de88044b1de as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/f71ae55e30ff42d6997b1de88044b1de 2024-11-20T06:42:02,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/f71ae55e30ff42d6997b1de88044b1de, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T06:42:02,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/1a6a46a815b34a59a80d7d88a702ef21 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1a6a46a815b34a59a80d7d88a702ef21 2024-11-20T06:42:02,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1a6a46a815b34a59a80d7d88a702ef21, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T06:42:02,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6bac99491880cd23064dd43f1153d648 in 842ms, sequenceid=154, compaction requested=true 2024-11-20T06:42:02,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:02,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:02,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:02,899 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:02,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:02,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:02,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:02,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T06:42:02,899 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:02,900 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41021 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:02,900 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:02,900 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/A is initiating minor compaction (all files) 2024-11-20T06:42:02,900 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/B is initiating minor compaction (all files) 2024-11-20T06:42:02,900 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/B in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,900 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/A in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,900 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/02c07510238448ea88e86efe664539b2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a65288cdb0674ffc8181394adc719226, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/f71ae55e30ff42d6997b1de88044b1de] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=35.4 K 2024-11-20T06:42:02,900 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/30ad491ee28240a5b913e8c3a1c1fabd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f4742057272245f3ab7aa4f1ed0aa8a2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/15adc96db91f452c9bc5da8240762326] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=40.1 K 2024-11-20T06:42:02,900 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30ad491ee28240a5b913e8c3a1c1fabd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732084918901 2024-11-20T06:42:02,900 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c07510238448ea88e86efe664539b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732084918901 2024-11-20T06:42:02,901 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4742057272245f3ab7aa4f1ed0aa8a2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732084920074 2024-11-20T06:42:02,901 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a65288cdb0674ffc8181394adc719226, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732084920074 2024-11-20T06:42:02,901 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15adc96db91f452c9bc5da8240762326, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084920890 2024-11-20T06:42:02,901 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting f71ae55e30ff42d6997b1de88044b1de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084920890 2024-11-20T06:42:02,906 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:02,906 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#A#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:02,907 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/674282bf068347b6a612a16d21b57d73 is 50, key is test_row_0/B:col10/1732084920890/Put/seqid=0 2024-11-20T06:42:02,907 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/e9aa8e55d5444088a7cbb54f232d3fc8 is 50, key is test_row_0/A:col10/1732084920890/Put/seqid=0 2024-11-20T06:42:02,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T06:42:02,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:02,907 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:42:02,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/024520ad9b1e43669fad54c687d672ea is 50, key is test_row_0/A:col10/1732084922098/Put/seqid=0 2024-11-20T06:42:02,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742287_1463 (size=12391) 2024-11-20T06:42:02,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742286_1462 (size=12391) 2024-11-20T06:42:02,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742288_1464 (size=12151) 2024-11-20T06:42:02,920 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/024520ad9b1e43669fad54c687d672ea 2024-11-20T06:42:02,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/e3d67f95f98a43528b5bb914d5e046d1 is 50, key is test_row_0/B:col10/1732084922098/Put/seqid=0 2024-11-20T06:42:02,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742289_1465 (size=12151) 2024-11-20T06:42:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:03,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:03,317 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/e9aa8e55d5444088a7cbb54f232d3fc8 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e9aa8e55d5444088a7cbb54f232d3fc8 2024-11-20T06:42:03,320 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/674282bf068347b6a612a16d21b57d73 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/674282bf068347b6a612a16d21b57d73 2024-11-20T06:42:03,321 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/A of 6bac99491880cd23064dd43f1153d648 into e9aa8e55d5444088a7cbb54f232d3fc8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:03,321 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:03,321 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/A, priority=13, startTime=1732084922899; duration=0sec 2024-11-20T06:42:03,321 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:03,321 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:A 2024-11-20T06:42:03,321 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:03,322 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:03,322 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/C is initiating minor compaction (all files) 2024-11-20T06:42:03,322 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/C in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:03,322 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/221b8a1c1b204400ae00ecebf2352c57, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/a8e394010d9e4560b0b36884d484985f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1a6a46a815b34a59a80d7d88a702ef21] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=35.4 K 2024-11-20T06:42:03,322 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221b8a1c1b204400ae00ecebf2352c57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732084918901 2024-11-20T06:42:03,323 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8e394010d9e4560b0b36884d484985f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732084920074 2024-11-20T06:42:03,323 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/B of 6bac99491880cd23064dd43f1153d648 into 674282bf068347b6a612a16d21b57d73(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:03,323 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:03,323 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/B, priority=13, startTime=1732084922899; duration=0sec 2024-11-20T06:42:03,323 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a6a46a815b34a59a80d7d88a702ef21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084920890 2024-11-20T06:42:03,323 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:03,323 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:B 2024-11-20T06:42:03,327 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#C#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:03,328 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/b0c52afa28924eb0858b8d164c532f86 is 50, key is test_row_0/C:col10/1732084920890/Put/seqid=0 2024-11-20T06:42:03,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742290_1466 (size=12391) 2024-11-20T06:42:03,333 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/e3d67f95f98a43528b5bb914d5e046d1 2024-11-20T06:42:03,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/c43acc9e3f14468daf710f3d49049fe5 is 50, key is test_row_0/C:col10/1732084922098/Put/seqid=0 2024-11-20T06:42:03,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742291_1467 (size=12151) 2024-11-20T06:42:03,342 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/c43acc9e3f14468daf710f3d49049fe5 2024-11-20T06:42:03,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/024520ad9b1e43669fad54c687d672ea as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/024520ad9b1e43669fad54c687d672ea 2024-11-20T06:42:03,348 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/024520ad9b1e43669fad54c687d672ea, entries=150, sequenceid=165, filesize=11.9 K 2024-11-20T06:42:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/e3d67f95f98a43528b5bb914d5e046d1 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/e3d67f95f98a43528b5bb914d5e046d1 2024-11-20T06:42:03,351 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/e3d67f95f98a43528b5bb914d5e046d1, entries=150, sequenceid=165, filesize=11.9 K 2024-11-20T06:42:03,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/c43acc9e3f14468daf710f3d49049fe5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c43acc9e3f14468daf710f3d49049fe5 2024-11-20T06:42:03,355 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c43acc9e3f14468daf710f3d49049fe5, entries=150, sequenceid=165, filesize=11.9 K 2024-11-20T06:42:03,355 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=93.93 KB/96180 for 6bac99491880cd23064dd43f1153d648 in 448ms, sequenceid=165, compaction requested=false 2024-11-20T06:42:03,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:03,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:03,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-20T06:42:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-20T06:42:03,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-20T06:42:03,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9900 sec 2024-11-20T06:42:03,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9940 sec 2024-11-20T06:42:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:03,361 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:03,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T06:42:03,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/dca3b9d892b44271acc535262fae0ba4 is 50, key is test_row_0/A:col10/1732084923348/Put/seqid=0 2024-11-20T06:42:03,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742292_1468 (size=14541) 2024-11-20T06:42:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084983457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T06:42:03,514 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:03,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-20T06:42:03,515 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T06:42:03,516 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:03,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:03,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084983564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:03,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T06:42:03,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T06:42:03,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:03,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:03,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:03,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:03,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:03,734 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/b0c52afa28924eb0858b8d164c532f86 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/b0c52afa28924eb0858b8d164c532f86 2024-11-20T06:42:03,737 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/C of 6bac99491880cd23064dd43f1153d648 into b0c52afa28924eb0858b8d164c532f86(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:03,737 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:03,737 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/C, priority=13, startTime=1732084922899; duration=0sec 2024-11-20T06:42:03,737 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:03,737 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:C 2024-11-20T06:42:03,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/dca3b9d892b44271acc535262fae0ba4 2024-11-20T06:42:03,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/014ca312cc7b49b8bd58c59b28e88885 is 50, key is test_row_0/B:col10/1732084923348/Put/seqid=0 2024-11-20T06:42:03,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742293_1469 (size=12151) 2024-11-20T06:42:03,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084983773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:03,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T06:42:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:03,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:03,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:03,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:03,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T06:42:03,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T06:42:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:03,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084984084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:04,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:04,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:04,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T06:42:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T06:42:04,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/014ca312cc7b49b8bd58c59b28e88885 2024-11-20T06:42:04,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/013ef13044e84f718093bb44d647b889 is 50, key is test_row_0/C:col10/1732084923348/Put/seqid=0 2024-11-20T06:42:04,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742294_1470 (size=12151) 2024-11-20T06:42:04,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T06:42:04,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:04,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T06:42:04,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:04,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T06:42:04,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:04,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:04,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/013ef13044e84f718093bb44d647b889 2024-11-20T06:42:04,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/dca3b9d892b44271acc535262fae0ba4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/dca3b9d892b44271acc535262fae0ba4 2024-11-20T06:42:04,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/dca3b9d892b44271acc535262fae0ba4, entries=200, sequenceid=185, filesize=14.2 K 2024-11-20T06:42:04,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/014ca312cc7b49b8bd58c59b28e88885 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/014ca312cc7b49b8bd58c59b28e88885 2024-11-20T06:42:04,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/014ca312cc7b49b8bd58c59b28e88885, entries=150, sequenceid=185, filesize=11.9 K 2024-11-20T06:42:04,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084984594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:04,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/013ef13044e84f718093bb44d647b889 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/013ef13044e84f718093bb44d647b889 2024-11-20T06:42:04,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/013ef13044e84f718093bb44d647b889, entries=150, sequenceid=185, filesize=11.9 K 2024-11-20T06:42:04,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 6bac99491880cd23064dd43f1153d648 in 1242ms, sequenceid=185, compaction requested=true 2024-11-20T06:42:04,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:04,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:04,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:04,604 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:04,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:04,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:04,604 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:04,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:04,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:04,604 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:04,604 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:04,604 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/A is initiating minor compaction (all files) 2024-11-20T06:42:04,604 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/B is initiating minor compaction (all files) 2024-11-20T06:42:04,604 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/A in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,604 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/B in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,604 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e9aa8e55d5444088a7cbb54f232d3fc8, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/024520ad9b1e43669fad54c687d672ea, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/dca3b9d892b44271acc535262fae0ba4] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=38.2 K 2024-11-20T06:42:04,604 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/674282bf068347b6a612a16d21b57d73, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/e3d67f95f98a43528b5bb914d5e046d1, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/014ca312cc7b49b8bd58c59b28e88885] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=35.8 K 2024-11-20T06:42:04,605 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9aa8e55d5444088a7cbb54f232d3fc8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084920890 2024-11-20T06:42:04,605 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 674282bf068347b6a612a16d21b57d73, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084920890 2024-11-20T06:42:04,605 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024520ad9b1e43669fad54c687d672ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732084922092 2024-11-20T06:42:04,605 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d67f95f98a43528b5bb914d5e046d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732084922092 2024-11-20T06:42:04,605 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca3b9d892b44271acc535262fae0ba4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732084923299 2024-11-20T06:42:04,605 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 014ca312cc7b49b8bd58c59b28e88885, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732084923314 2024-11-20T06:42:04,611 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#A#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:04,611 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#B#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:04,612 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/2b079ec8e0f9471f985cc1f19e3e3d33 is 50, key is test_row_0/B:col10/1732084923348/Put/seqid=0 2024-11-20T06:42:04,612 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/bcd664ddb13b49d8bc654965e68dc55f is 50, key is test_row_0/A:col10/1732084923348/Put/seqid=0 2024-11-20T06:42:04,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742295_1471 (size=12493) 2024-11-20T06:42:04,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742296_1472 (size=12493) 2024-11-20T06:42:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T06:42:04,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T06:42:04,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:04,737 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-20T06:42:04,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/94990ba96051423cb824555c3454a8e2 is 50, key is test_row_0/A:col10/1732084923426/Put/seqid=0 2024-11-20T06:42:04,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742297_1473 (size=12151) 2024-11-20T06:42:04,743 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/94990ba96051423cb824555c3454a8e2 2024-11-20T06:42:04,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/b64728d210cf4969a73f62a42c67a69a is 50, key is test_row_0/B:col10/1732084923426/Put/seqid=0 2024-11-20T06:42:04,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742298_1474 (size=12151) 2024-11-20T06:42:05,024 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/bcd664ddb13b49d8bc654965e68dc55f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/bcd664ddb13b49d8bc654965e68dc55f 2024-11-20T06:42:05,025 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/2b079ec8e0f9471f985cc1f19e3e3d33 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/2b079ec8e0f9471f985cc1f19e3e3d33 2024-11-20T06:42:05,028 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/A of 6bac99491880cd23064dd43f1153d648 into bcd664ddb13b49d8bc654965e68dc55f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:05,028 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:05,028 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/A, priority=13, startTime=1732084924604; duration=0sec 2024-11-20T06:42:05,028 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:05,028 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:A 2024-11-20T06:42:05,028 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:05,028 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/B of 6bac99491880cd23064dd43f1153d648 into 2b079ec8e0f9471f985cc1f19e3e3d33(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:05,028 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:05,029 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/B, priority=13, startTime=1732084924604; duration=0sec 2024-11-20T06:42:05,029 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:05,029 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:B 2024-11-20T06:42:05,030 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:05,030 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/C is initiating minor compaction (all files) 2024-11-20T06:42:05,030 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/C in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:05,030 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/b0c52afa28924eb0858b8d164c532f86, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c43acc9e3f14468daf710f3d49049fe5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/013ef13044e84f718093bb44d647b889] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=35.8 K 2024-11-20T06:42:05,030 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c52afa28924eb0858b8d164c532f86, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732084920890 2024-11-20T06:42:05,030 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting c43acc9e3f14468daf710f3d49049fe5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732084922092 2024-11-20T06:42:05,030 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 013ef13044e84f718093bb44d647b889, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732084923314 2024-11-20T06:42:05,036 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#C#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:05,036 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/04079c997f504c2bbf029bdc1cd3923c is 50, key is test_row_0/C:col10/1732084923348/Put/seqid=0 2024-11-20T06:42:05,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742299_1475 (size=12493) 2024-11-20T06:42:05,046 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/04079c997f504c2bbf029bdc1cd3923c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/04079c997f504c2bbf029bdc1cd3923c 2024-11-20T06:42:05,050 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/C of 6bac99491880cd23064dd43f1153d648 into 04079c997f504c2bbf029bdc1cd3923c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:05,050 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:05,050 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/C, priority=13, startTime=1732084924604; duration=0sec 2024-11-20T06:42:05,050 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:05,050 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:C 2024-11-20T06:42:05,162 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/b64728d210cf4969a73f62a42c67a69a 2024-11-20T06:42:05,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/07586265b7c048efb6389b7399d01824 is 50, key is test_row_0/C:col10/1732084923426/Put/seqid=0 2024-11-20T06:42:05,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742300_1476 (size=12151) 2024-11-20T06:42:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:05,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:05,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084985515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084985516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084985520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084985522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,571 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/07586265b7c048efb6389b7399d01824 2024-11-20T06:42:05,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/94990ba96051423cb824555c3454a8e2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/94990ba96051423cb824555c3454a8e2 2024-11-20T06:42:05,577 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/94990ba96051423cb824555c3454a8e2, entries=150, sequenceid=204, filesize=11.9 K 2024-11-20T06:42:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/b64728d210cf4969a73f62a42c67a69a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/b64728d210cf4969a73f62a42c67a69a 2024-11-20T06:42:05,580 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/b64728d210cf4969a73f62a42c67a69a, entries=150, sequenceid=204, filesize=11.9 K 2024-11-20T06:42:05,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/07586265b7c048efb6389b7399d01824 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/07586265b7c048efb6389b7399d01824 2024-11-20T06:42:05,583 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/07586265b7c048efb6389b7399d01824, entries=150, sequenceid=204, filesize=11.9 K 2024-11-20T06:42:05,584 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 6bac99491880cd23064dd43f1153d648 in 847ms, sequenceid=204, compaction requested=false 2024-11-20T06:42:05,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:05,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:05,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-20T06:42:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-20T06:42:05,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-20T06:42:05,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0690 sec 2024-11-20T06:42:05,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0730 sec 2024-11-20T06:42:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:05,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:05,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-20T06:42:05,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/4893feff287d44989fd77e7c865dd166 is 50, key is test_row_0/A:col10/1732084925515/Put/seqid=0 2024-11-20T06:42:05,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742301_1477 (size=14541) 2024-11-20T06:42:05,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084985647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084985651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084985652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T06:42:05,664 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:05,664 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-20T06:42:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T06:42:05,665 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:05,666 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:05,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:05,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084985654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084985655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084985763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084985763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084985763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T06:42:05,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084985774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084985774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T06:42:05,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:05,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:05,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:05,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:05,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:05,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T06:42:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:05,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084985973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084985974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084985974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T06:42:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084985984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084985984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/4893feff287d44989fd77e7c865dd166 2024-11-20T06:42:06,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/18f49dcf6b0d4a918f40e5f69cc1ce81 is 50, key is test_row_0/B:col10/1732084925515/Put/seqid=0 2024-11-20T06:42:06,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742302_1478 (size=12151) 2024-11-20T06:42:06,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T06:42:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T06:42:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084986283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T06:42:06,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084986294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:06,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084986294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084986294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:06,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084986294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/18f49dcf6b0d4a918f40e5f69cc1ce81 2024-11-20T06:42:06,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T06:42:06,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/91b9b1d1c5174281a7858c63cb0e63c8 is 50, key is test_row_0/C:col10/1732084925515/Put/seqid=0 2024-11-20T06:42:06,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742303_1479 (size=12151) 2024-11-20T06:42:06,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T06:42:06,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:06,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T06:42:06,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:06,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084986794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T06:42:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084986804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084986804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084986813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:06,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084986813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:06,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/91b9b1d1c5174281a7858c63cb0e63c8 2024-11-20T06:42:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/4893feff287d44989fd77e7c865dd166 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/4893feff287d44989fd77e7c865dd166 2024-11-20T06:42:06,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/4893feff287d44989fd77e7c865dd166, entries=200, sequenceid=226, filesize=14.2 K 2024-11-20T06:42:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/18f49dcf6b0d4a918f40e5f69cc1ce81 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/18f49dcf6b0d4a918f40e5f69cc1ce81 2024-11-20T06:42:06,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/18f49dcf6b0d4a918f40e5f69cc1ce81, entries=150, sequenceid=226, filesize=11.9 K 2024-11-20T06:42:06,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/91b9b1d1c5174281a7858c63cb0e63c8 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/91b9b1d1c5174281a7858c63cb0e63c8 2024-11-20T06:42:06,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/91b9b1d1c5174281a7858c63cb0e63c8, entries=150, sequenceid=226, filesize=11.9 K 2024-11-20T06:42:06,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 6bac99491880cd23064dd43f1153d648 in 1241ms, sequenceid=226, compaction requested=true 2024-11-20T06:42:06,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:06,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:06,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:06,851 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:06,851 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:06,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:06,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:06,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:06,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:06,852 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:06,852 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:06,852 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/B is initiating minor compaction (all files) 2024-11-20T06:42:06,852 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/A is initiating minor compaction (all files) 2024-11-20T06:42:06,852 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/A in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,852 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/B in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,852 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/2b079ec8e0f9471f985cc1f19e3e3d33, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/b64728d210cf4969a73f62a42c67a69a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/18f49dcf6b0d4a918f40e5f69cc1ce81] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=35.9 K 2024-11-20T06:42:06,852 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/bcd664ddb13b49d8bc654965e68dc55f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/94990ba96051423cb824555c3454a8e2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/4893feff287d44989fd77e7c865dd166] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=38.3 K 2024-11-20T06:42:06,853 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b079ec8e0f9471f985cc1f19e3e3d33, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732084923314 2024-11-20T06:42:06,853 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcd664ddb13b49d8bc654965e68dc55f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732084923314 2024-11-20T06:42:06,853 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94990ba96051423cb824555c3454a8e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732084923426 2024-11-20T06:42:06,853 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b64728d210cf4969a73f62a42c67a69a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732084923426 2024-11-20T06:42:06,853 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4893feff287d44989fd77e7c865dd166, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732084925491 2024-11-20T06:42:06,853 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 18f49dcf6b0d4a918f40e5f69cc1ce81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732084925515 2024-11-20T06:42:06,858 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:06,858 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:06,858 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/f301733f36e74b24ba346f373be71e8e is 50, key is test_row_0/A:col10/1732084925515/Put/seqid=0 2024-11-20T06:42:06,858 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/4f1aa34035c148b5bf0b81759f7e365d is 50, key is test_row_0/B:col10/1732084925515/Put/seqid=0 2024-11-20T06:42:06,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742304_1480 (size=12595) 2024-11-20T06:42:06,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742305_1481 (size=12595) 2024-11-20T06:42:06,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T06:42:06,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:06,887 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T06:42:06,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/ad7f70b35de14c57978c470b425252c7 is 50, key is test_row_0/A:col10/1732084925654/Put/seqid=0 2024-11-20T06:42:06,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742306_1482 (size=12151) 2024-11-20T06:42:07,274 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/4f1aa34035c148b5bf0b81759f7e365d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/4f1aa34035c148b5bf0b81759f7e365d 2024-11-20T06:42:07,279 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/B of 6bac99491880cd23064dd43f1153d648 into 4f1aa34035c148b5bf0b81759f7e365d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:07,279 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:07,279 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/B, priority=13, startTime=1732084926851; duration=0sec 2024-11-20T06:42:07,279 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:07,279 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:B 2024-11-20T06:42:07,279 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:07,280 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/f301733f36e74b24ba346f373be71e8e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f301733f36e74b24ba346f373be71e8e 2024-11-20T06:42:07,280 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:07,280 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/C is initiating minor compaction (all files) 2024-11-20T06:42:07,280 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/C in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:07,280 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/04079c997f504c2bbf029bdc1cd3923c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/07586265b7c048efb6389b7399d01824, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/91b9b1d1c5174281a7858c63cb0e63c8] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=35.9 K 2024-11-20T06:42:07,281 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 04079c997f504c2bbf029bdc1cd3923c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732084923314 2024-11-20T06:42:07,281 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 07586265b7c048efb6389b7399d01824, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732084923426 2024-11-20T06:42:07,282 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 91b9b1d1c5174281a7858c63cb0e63c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732084925515 2024-11-20T06:42:07,284 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/A of 6bac99491880cd23064dd43f1153d648 into f301733f36e74b24ba346f373be71e8e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:07,284 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:07,284 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/A, priority=13, startTime=1732084926851; duration=0sec 2024-11-20T06:42:07,284 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:07,284 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:A 2024-11-20T06:42:07,287 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:07,287 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/271600ce9ae249fb9505f6d51bc8f0b3 is 50, key is test_row_0/C:col10/1732084925515/Put/seqid=0 2024-11-20T06:42:07,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742307_1483 (size=12595) 2024-11-20T06:42:07,293 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/ad7f70b35de14c57978c470b425252c7 2024-11-20T06:42:07,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/9c5248f4e2284a8fb0b2eb19878f25c5 is 50, key is test_row_0/B:col10/1732084925654/Put/seqid=0 2024-11-20T06:42:07,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742308_1484 (size=12151) 2024-11-20T06:42:07,696 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/271600ce9ae249fb9505f6d51bc8f0b3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/271600ce9ae249fb9505f6d51bc8f0b3 2024-11-20T06:42:07,700 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/C of 6bac99491880cd23064dd43f1153d648 into 271600ce9ae249fb9505f6d51bc8f0b3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:07,700 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:07,700 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/C, priority=13, startTime=1732084926852; duration=0sec 2024-11-20T06:42:07,700 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:07,700 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:C 2024-11-20T06:42:07,702 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/9c5248f4e2284a8fb0b2eb19878f25c5 2024-11-20T06:42:07,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/5f9e53b9afd641d1bf7cc89a7a034654 is 50, key is test_row_0/C:col10/1732084925654/Put/seqid=0 2024-11-20T06:42:07,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742309_1485 (size=12151) 2024-11-20T06:42:07,715 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/5f9e53b9afd641d1bf7cc89a7a034654 2024-11-20T06:42:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/ad7f70b35de14c57978c470b425252c7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/ad7f70b35de14c57978c470b425252c7 2024-11-20T06:42:07,724 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/ad7f70b35de14c57978c470b425252c7, entries=150, sequenceid=243, filesize=11.9 K 2024-11-20T06:42:07,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/9c5248f4e2284a8fb0b2eb19878f25c5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/9c5248f4e2284a8fb0b2eb19878f25c5 2024-11-20T06:42:07,729 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/9c5248f4e2284a8fb0b2eb19878f25c5, entries=150, sequenceid=243, filesize=11.9 K 2024-11-20T06:42:07,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/5f9e53b9afd641d1bf7cc89a7a034654 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/5f9e53b9afd641d1bf7cc89a7a034654 2024-11-20T06:42:07,736 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/5f9e53b9afd641d1bf7cc89a7a034654, entries=150, sequenceid=243, filesize=11.9 K 2024-11-20T06:42:07,736 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 6bac99491880cd23064dd43f1153d648 in 849ms, sequenceid=243, compaction requested=false 2024-11-20T06:42:07,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:07,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:07,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-20T06:42:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-20T06:42:07,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-20T06:42:07,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0730 sec 2024-11-20T06:42:07,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.0780 sec 2024-11-20T06:42:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T06:42:07,814 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:07,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-20T06:42:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T06:42:07,817 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:07,817 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:07,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:07,822 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:07,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:42:07,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/3d02e6b772134cf994b4129f5e4c1abe is 50, key is test_row_0/A:col10/1732084927819/Put/seqid=0 2024-11-20T06:42:07,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742310_1486 (size=16981) 2024-11-20T06:42:07,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:07,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084987857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084987858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:07,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084987861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:07,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084987890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084987890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:07,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T06:42:07,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T06:42:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:07,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:07,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:07,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084987994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:07,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084987994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084987994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084988004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084988004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T06:42:08,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:08,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T06:42:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084988204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084988204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084988204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084988213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084988213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/3d02e6b772134cf994b4129f5e4c1abe 2024-11-20T06:42:08,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/3a4ffcfae6a5456a877784acc2b66fa5 is 50, key is test_row_0/B:col10/1732084927819/Put/seqid=0 2024-11-20T06:42:08,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742311_1487 (size=12201) 2024-11-20T06:42:08,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T06:42:08,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:08,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T06:42:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T06:42:08,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084988514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084988514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084988514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084988534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084988534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:08,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:08,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T06:42:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/3a4ffcfae6a5456a877784acc2b66fa5 2024-11-20T06:42:08,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/6ecdb66c8e88465c873718ac06d77c00 is 50, key is test_row_0/C:col10/1732084927819/Put/seqid=0 2024-11-20T06:42:08,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742312_1488 (size=12201) 2024-11-20T06:42:08,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T06:42:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T06:42:08,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:08,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:08,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T06:42:09,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084989024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:09,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084989024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:09,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084989024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:09,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:09,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:09,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T06:42:09,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:09,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:09,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:09,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:09,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:09,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084989043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:09,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084989043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:09,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/6ecdb66c8e88465c873718ac06d77c00 2024-11-20T06:42:09,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/3d02e6b772134cf994b4129f5e4c1abe as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3d02e6b772134cf994b4129f5e4c1abe 2024-11-20T06:42:09,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3d02e6b772134cf994b4129f5e4c1abe, entries=250, sequenceid=257, filesize=16.6 K 2024-11-20T06:42:09,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/3a4ffcfae6a5456a877784acc2b66fa5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/3a4ffcfae6a5456a877784acc2b66fa5 2024-11-20T06:42:09,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/3a4ffcfae6a5456a877784acc2b66fa5, entries=150, sequenceid=257, filesize=11.9 K 2024-11-20T06:42:09,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/6ecdb66c8e88465c873718ac06d77c00 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6ecdb66c8e88465c873718ac06d77c00 2024-11-20T06:42:09,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6ecdb66c8e88465c873718ac06d77c00, entries=150, sequenceid=257, filesize=11.9 K 2024-11-20T06:42:09,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6bac99491880cd23064dd43f1153d648 in 1243ms, sequenceid=257, compaction requested=true 2024-11-20T06:42:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:09,065 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:09,065 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:09,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:09,066 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:09,066 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:09,066 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/B is initiating minor compaction (all files) 2024-11-20T06:42:09,066 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/A is initiating minor compaction (all files) 2024-11-20T06:42:09,066 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/A in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:09,066 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/B in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:09,066 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/4f1aa34035c148b5bf0b81759f7e365d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/9c5248f4e2284a8fb0b2eb19878f25c5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/3a4ffcfae6a5456a877784acc2b66fa5] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=36.1 K 2024-11-20T06:42:09,066 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f301733f36e74b24ba346f373be71e8e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/ad7f70b35de14c57978c470b425252c7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3d02e6b772134cf994b4129f5e4c1abe] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=40.7 K 2024-11-20T06:42:09,066 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f1aa34035c148b5bf0b81759f7e365d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732084925515 2024-11-20T06:42:09,066 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting f301733f36e74b24ba346f373be71e8e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732084925515 2024-11-20T06:42:09,067 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7f70b35de14c57978c470b425252c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732084925650 2024-11-20T06:42:09,067 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c5248f4e2284a8fb0b2eb19878f25c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732084925650 2024-11-20T06:42:09,067 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d02e6b772134cf994b4129f5e4c1abe, keycount=250, bloomtype=ROW, size=16.6 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084927814 2024-11-20T06:42:09,067 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4ffcfae6a5456a877784acc2b66fa5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084927819 2024-11-20T06:42:09,072 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:09,072 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/ba9c98b44c3141cf8abc3daf0af2388e is 50, key is test_row_0/B:col10/1732084927819/Put/seqid=0 2024-11-20T06:42:09,075 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:09,075 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/3247b4e70541428884467d1963f3b51e is 50, key is test_row_0/A:col10/1732084927819/Put/seqid=0 2024-11-20T06:42:09,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742313_1489 (size=12747) 2024-11-20T06:42:09,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742314_1490 (size=12747) 2024-11-20T06:42:09,084 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/3247b4e70541428884467d1963f3b51e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3247b4e70541428884467d1963f3b51e 2024-11-20T06:42:09,088 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/A of 6bac99491880cd23064dd43f1153d648 into 3247b4e70541428884467d1963f3b51e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:09,088 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:09,088 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/A, priority=13, startTime=1732084929065; duration=0sec 2024-11-20T06:42:09,088 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:09,088 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:A 2024-11-20T06:42:09,088 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:09,089 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:09,089 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/C is initiating minor compaction (all files) 2024-11-20T06:42:09,089 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/C in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:09,089 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/271600ce9ae249fb9505f6d51bc8f0b3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/5f9e53b9afd641d1bf7cc89a7a034654, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6ecdb66c8e88465c873718ac06d77c00] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=36.1 K 2024-11-20T06:42:09,090 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 271600ce9ae249fb9505f6d51bc8f0b3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732084925515 2024-11-20T06:42:09,091 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f9e53b9afd641d1bf7cc89a7a034654, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732084925650 2024-11-20T06:42:09,091 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ecdb66c8e88465c873718ac06d77c00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084927819 2024-11-20T06:42:09,096 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:09,096 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/c0552c66914b4318892c3e41eda3a332 is 50, key is test_row_0/C:col10/1732084927819/Put/seqid=0 2024-11-20T06:42:09,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742315_1491 (size=12747) 2024-11-20T06:42:09,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T06:42:09,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:09,189 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T06:42:09,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/6b8edf9f9ab343f48361f2fa9054eec5 is 50, key is test_row_0/A:col10/1732084927861/Put/seqid=0 2024-11-20T06:42:09,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742316_1492 (size=12301) 2024-11-20T06:42:09,197 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/6b8edf9f9ab343f48361f2fa9054eec5 2024-11-20T06:42:09,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/5a060e2b37764b429e289b1388c87bc0 is 50, key is test_row_0/B:col10/1732084927861/Put/seqid=0 2024-11-20T06:42:09,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742317_1493 (size=12301) 2024-11-20T06:42:09,482 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/ba9c98b44c3141cf8abc3daf0af2388e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/ba9c98b44c3141cf8abc3daf0af2388e 2024-11-20T06:42:09,485 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/B of 6bac99491880cd23064dd43f1153d648 into ba9c98b44c3141cf8abc3daf0af2388e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:09,485 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:09,485 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/B, priority=13, startTime=1732084929065; duration=0sec 2024-11-20T06:42:09,485 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:09,485 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:B 2024-11-20T06:42:09,503 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/c0552c66914b4318892c3e41eda3a332 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c0552c66914b4318892c3e41eda3a332 2024-11-20T06:42:09,506 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/C of 6bac99491880cd23064dd43f1153d648 into c0552c66914b4318892c3e41eda3a332(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:09,506 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:09,506 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/C, priority=13, startTime=1732084929065; duration=0sec 2024-11-20T06:42:09,506 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:09,506 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:C 2024-11-20T06:42:09,607 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/5a060e2b37764b429e289b1388c87bc0 2024-11-20T06:42:09,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/1b03fdecd02b4eac930532bddcb0e6f7 is 50, key is test_row_0/C:col10/1732084927861/Put/seqid=0 2024-11-20T06:42:09,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742318_1494 (size=12301) 2024-11-20T06:42:09,618 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/1b03fdecd02b4eac930532bddcb0e6f7 2024-11-20T06:42:09,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/6b8edf9f9ab343f48361f2fa9054eec5 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6b8edf9f9ab343f48361f2fa9054eec5 2024-11-20T06:42:09,624 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6b8edf9f9ab343f48361f2fa9054eec5, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T06:42:09,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/5a060e2b37764b429e289b1388c87bc0 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5a060e2b37764b429e289b1388c87bc0 2024-11-20T06:42:09,627 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5a060e2b37764b429e289b1388c87bc0, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T06:42:09,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/1b03fdecd02b4eac930532bddcb0e6f7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1b03fdecd02b4eac930532bddcb0e6f7 2024-11-20T06:42:09,630 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1b03fdecd02b4eac930532bddcb0e6f7, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T06:42:09,631 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 6bac99491880cd23064dd43f1153d648 in 442ms, sequenceid=284, compaction requested=false 2024-11-20T06:42:09,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:09,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:09,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-20T06:42:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-20T06:42:09,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-20T06:42:09,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-11-20T06:42:09,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.8200 sec 2024-11-20T06:42:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T06:42:09,964 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:09,964 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-20T06:42:09,966 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T06:42:09,966 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:09,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:10,042 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:10,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:42:10,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/da9646e36bf4413e850255fdcf31c709 is 50, key is test_row_0/A:col10/1732084930038/Put/seqid=0 2024-11-20T06:42:10,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742319_1495 (size=22065) 2024-11-20T06:42:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T06:42:10,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084990115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084990115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T06:42:10,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:10,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084990116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084990117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084990117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084990223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084990223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084990234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084990234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084990234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T06:42:10,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:10,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T06:42:10,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T06:42:10,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:10,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084990434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084990434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/da9646e36bf4413e850255fdcf31c709 2024-11-20T06:42:10,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/245fb7f72c97495b940ecd640ff1f0f1 is 50, key is test_row_0/B:col10/1732084930038/Put/seqid=0 2024-11-20T06:42:10,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084990444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084990444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084990444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742320_1496 (size=12301) 2024-11-20T06:42:10,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T06:42:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T06:42:10,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T06:42:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084990743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084990754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084990764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084990764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084990764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:10,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/245fb7f72c97495b940ecd640ff1f0f1 2024-11-20T06:42:10,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/43ac2f747f98494eab9745e727f687bd is 50, key is test_row_0/C:col10/1732084930038/Put/seqid=0 2024-11-20T06:42:10,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742321_1497 (size=12301) 2024-11-20T06:42:10,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T06:42:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:10,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:11,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T06:42:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:11,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T06:42:11,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T06:42:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:11,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:11,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:11,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084991254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:11,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:11,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:11,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:11,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084991263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:11,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/43ac2f747f98494eab9745e727f687bd 2024-11-20T06:42:11,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084991273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:11,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/da9646e36bf4413e850255fdcf31c709 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/da9646e36bf4413e850255fdcf31c709 2024-11-20T06:42:11,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/da9646e36bf4413e850255fdcf31c709, entries=350, sequenceid=297, filesize=21.5 K 2024-11-20T06:42:11,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/245fb7f72c97495b940ecd640ff1f0f1 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/245fb7f72c97495b940ecd640ff1f0f1 2024-11-20T06:42:11,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/245fb7f72c97495b940ecd640ff1f0f1, entries=150, sequenceid=297, filesize=12.0 K 2024-11-20T06:42:11,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/43ac2f747f98494eab9745e727f687bd as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/43ac2f747f98494eab9745e727f687bd 2024-11-20T06:42:11,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084991284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:11,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:11,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084991284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:11,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:11,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/43ac2f747f98494eab9745e727f687bd, entries=150, sequenceid=297, filesize=12.0 K 2024-11-20T06:42:11,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6bac99491880cd23064dd43f1153d648 in 1252ms, sequenceid=297, compaction requested=true 2024-11-20T06:42:11,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:11,294 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:11,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:11,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:11,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:11,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:11,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:11,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T06:42:11,295 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:11,295 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47113 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:11,295 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/A is initiating minor compaction (all files) 2024-11-20T06:42:11,295 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/A in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:11,295 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3247b4e70541428884467d1963f3b51e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6b8edf9f9ab343f48361f2fa9054eec5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/da9646e36bf4413e850255fdcf31c709] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=46.0 K 2024-11-20T06:42:11,296 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:11,296 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/B is initiating minor compaction (all files) 2024-11-20T06:42:11,296 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3247b4e70541428884467d1963f3b51e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084927819 2024-11-20T06:42:11,296 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/B in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:11,296 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/ba9c98b44c3141cf8abc3daf0af2388e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5a060e2b37764b429e289b1388c87bc0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/245fb7f72c97495b940ecd640ff1f0f1] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=36.5 K 2024-11-20T06:42:11,296 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b8edf9f9ab343f48361f2fa9054eec5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732084927860 2024-11-20T06:42:11,296 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9c98b44c3141cf8abc3daf0af2388e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084927819 2024-11-20T06:42:11,296 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting da9646e36bf4413e850255fdcf31c709, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732084930034 2024-11-20T06:42:11,296 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a060e2b37764b429e289b1388c87bc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732084927860 2024-11-20T06:42:11,297 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 245fb7f72c97495b940ecd640ff1f0f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732084930038 2024-11-20T06:42:11,303 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#A#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:11,303 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/250882179c3b4903b1f29aaebfc49ce4 is 50, key is test_row_0/A:col10/1732084930038/Put/seqid=0 2024-11-20T06:42:11,305 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:11,306 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/4dc1675c98bb443eb5c659ca03d6eb65 is 50, key is test_row_0/B:col10/1732084930038/Put/seqid=0 2024-11-20T06:42:11,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742322_1498 (size=12949) 2024-11-20T06:42:11,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742323_1499 (size=12949) 2024-11-20T06:42:11,327 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/250882179c3b4903b1f29aaebfc49ce4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/250882179c3b4903b1f29aaebfc49ce4 2024-11-20T06:42:11,331 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/A of 6bac99491880cd23064dd43f1153d648 into 250882179c3b4903b1f29aaebfc49ce4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:11,331 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:11,331 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/A, priority=13, startTime=1732084931294; duration=0sec 2024-11-20T06:42:11,331 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:11,331 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:A 2024-11-20T06:42:11,331 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:11,332 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:11,332 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/C is initiating minor compaction (all files) 2024-11-20T06:42:11,332 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/C in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:11,332 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c0552c66914b4318892c3e41eda3a332, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1b03fdecd02b4eac930532bddcb0e6f7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/43ac2f747f98494eab9745e727f687bd] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=36.5 K 2024-11-20T06:42:11,332 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0552c66914b4318892c3e41eda3a332, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732084927819 2024-11-20T06:42:11,332 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b03fdecd02b4eac930532bddcb0e6f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732084927860 2024-11-20T06:42:11,333 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43ac2f747f98494eab9745e727f687bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732084930038 2024-11-20T06:42:11,337 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:11,337 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/348ff2ceebcc4d5e8a8f310fc1f4b7ac is 50, key is test_row_0/C:col10/1732084930038/Put/seqid=0 2024-11-20T06:42:11,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742324_1500 (size=12949) 2024-11-20T06:42:11,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T06:42:11,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:11,343 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T06:42:11,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/2b2aebae173b4becbcec7a059e346069 is 50, key is test_row_0/A:col10/1732084930114/Put/seqid=0 2024-11-20T06:42:11,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742325_1501 (size=12301) 2024-11-20T06:42:11,718 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/4dc1675c98bb443eb5c659ca03d6eb65 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/4dc1675c98bb443eb5c659ca03d6eb65 2024-11-20T06:42:11,721 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/B of 6bac99491880cd23064dd43f1153d648 into 4dc1675c98bb443eb5c659ca03d6eb65(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:11,721 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:11,722 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/B, priority=13, startTime=1732084931295; duration=0sec 2024-11-20T06:42:11,722 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:11,722 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:B 2024-11-20T06:42:11,744 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/348ff2ceebcc4d5e8a8f310fc1f4b7ac as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/348ff2ceebcc4d5e8a8f310fc1f4b7ac 2024-11-20T06:42:11,748 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/C of 6bac99491880cd23064dd43f1153d648 into 348ff2ceebcc4d5e8a8f310fc1f4b7ac(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:11,748 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:11,748 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/C, priority=13, startTime=1732084931295; duration=0sec 2024-11-20T06:42:11,748 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:11,748 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:C 2024-11-20T06:42:11,749 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/2b2aebae173b4becbcec7a059e346069 2024-11-20T06:42:11,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/985a6de12db34f6e8d1dd1fa27f3b850 is 50, key is test_row_0/B:col10/1732084930114/Put/seqid=0 2024-11-20T06:42:11,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742326_1502 (size=12301) 2024-11-20T06:42:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T06:42:12,160 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/985a6de12db34f6e8d1dd1fa27f3b850 2024-11-20T06:42:12,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/8db9662c15b14240b8119091a476e559 is 50, key is test_row_0/C:col10/1732084930114/Put/seqid=0 2024-11-20T06:42:12,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742327_1503 (size=12301) 2024-11-20T06:42:12,211 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/8db9662c15b14240b8119091a476e559 2024-11-20T06:42:12,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/2b2aebae173b4becbcec7a059e346069 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/2b2aebae173b4becbcec7a059e346069 2024-11-20T06:42:12,216 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/2b2aebae173b4becbcec7a059e346069, entries=150, sequenceid=323, filesize=12.0 K 2024-11-20T06:42:12,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/985a6de12db34f6e8d1dd1fa27f3b850 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/985a6de12db34f6e8d1dd1fa27f3b850 2024-11-20T06:42:12,219 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/985a6de12db34f6e8d1dd1fa27f3b850, entries=150, sequenceid=323, filesize=12.0 K 2024-11-20T06:42:12,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/8db9662c15b14240b8119091a476e559 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/8db9662c15b14240b8119091a476e559 2024-11-20T06:42:12,222 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/8db9662c15b14240b8119091a476e559, entries=150, sequenceid=323, filesize=12.0 K 2024-11-20T06:42:12,223 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 6bac99491880cd23064dd43f1153d648 in 880ms, sequenceid=323, compaction requested=false 2024-11-20T06:42:12,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:12,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:12,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-20T06:42:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-20T06:42:12,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-20T06:42:12,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2580 sec 2024-11-20T06:42:12,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.2620 sec 2024-11-20T06:42:12,282 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:12,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:42:12,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/6c5ee5e2cd944f85b8c8db38540a2d04 is 50, key is test_row_0/A:col10/1732084932276/Put/seqid=0 2024-11-20T06:42:12,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742328_1504 (size=14741) 2024-11-20T06:42:12,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084992338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084992340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084992341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084992344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084992345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084992454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084992454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084992454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084992454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084992464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084992664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084992664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084992664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084992664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084992674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/6c5ee5e2cd944f85b8c8db38540a2d04 2024-11-20T06:42:12,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/cd64fa58b91f403bbee8859eb8c33dbe is 50, key is test_row_0/B:col10/1732084932276/Put/seqid=0 2024-11-20T06:42:12,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742329_1505 (size=12301) 2024-11-20T06:42:12,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/cd64fa58b91f403bbee8859eb8c33dbe 2024-11-20T06:42:12,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/471a424f0c6d4470876c9cb0d554f8e8 is 50, key is test_row_0/C:col10/1732084932276/Put/seqid=0 2024-11-20T06:42:12,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742330_1506 (size=12301) 2024-11-20T06:42:12,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084992984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084992984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084992984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084992984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:12,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084992993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:12,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/471a424f0c6d4470876c9cb0d554f8e8 2024-11-20T06:42:13,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/6c5ee5e2cd944f85b8c8db38540a2d04 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6c5ee5e2cd944f85b8c8db38540a2d04 2024-11-20T06:42:13,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6c5ee5e2cd944f85b8c8db38540a2d04, entries=200, sequenceid=336, filesize=14.4 K 2024-11-20T06:42:13,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/cd64fa58b91f403bbee8859eb8c33dbe as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/cd64fa58b91f403bbee8859eb8c33dbe 2024-11-20T06:42:13,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/cd64fa58b91f403bbee8859eb8c33dbe, entries=150, sequenceid=336, filesize=12.0 K 2024-11-20T06:42:13,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/471a424f0c6d4470876c9cb0d554f8e8 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/471a424f0c6d4470876c9cb0d554f8e8 2024-11-20T06:42:13,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/471a424f0c6d4470876c9cb0d554f8e8, entries=150, sequenceid=336, filesize=12.0 K 2024-11-20T06:42:13,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6bac99491880cd23064dd43f1153d648 in 841ms, sequenceid=336, compaction requested=true 2024-11-20T06:42:13,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:13,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:13,123 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:13,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:13,123 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:13,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:13,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:13,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6bac99491880cd23064dd43f1153d648:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:13,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:13,124 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:13,124 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/A is initiating minor compaction (all files) 2024-11-20T06:42:13,124 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/A in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:13,124 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/250882179c3b4903b1f29aaebfc49ce4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/2b2aebae173b4becbcec7a059e346069, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6c5ee5e2cd944f85b8c8db38540a2d04] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=39.1 K 2024-11-20T06:42:13,124 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:13,124 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/B is initiating minor compaction (all files) 2024-11-20T06:42:13,124 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/B in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:13,124 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/4dc1675c98bb443eb5c659ca03d6eb65, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/985a6de12db34f6e8d1dd1fa27f3b850, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/cd64fa58b91f403bbee8859eb8c33dbe] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=36.7 K 2024-11-20T06:42:13,125 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dc1675c98bb443eb5c659ca03d6eb65, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732084930038 2024-11-20T06:42:13,125 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 250882179c3b4903b1f29aaebfc49ce4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732084930038 2024-11-20T06:42:13,125 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 985a6de12db34f6e8d1dd1fa27f3b850, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732084930114 2024-11-20T06:42:13,125 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b2aebae173b4becbcec7a059e346069, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732084930114 2024-11-20T06:42:13,125 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting cd64fa58b91f403bbee8859eb8c33dbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732084932276 2024-11-20T06:42:13,125 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c5ee5e2cd944f85b8c8db38540a2d04, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732084932274 2024-11-20T06:42:13,132 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#B#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:13,132 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#A#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:13,132 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/e6901b1c19b34809aba2db8513297903 is 50, key is test_row_0/A:col10/1732084932276/Put/seqid=0 2024-11-20T06:42:13,132 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/cb632a7926af46c591631a9203c84e5f is 50, key is test_row_0/B:col10/1732084932276/Put/seqid=0 2024-11-20T06:42:13,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742331_1507 (size=13051) 2024-11-20T06:42:13,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742332_1508 (size=13051) 2024-11-20T06:42:13,151 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/cb632a7926af46c591631a9203c84e5f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/cb632a7926af46c591631a9203c84e5f 2024-11-20T06:42:13,158 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/B of 6bac99491880cd23064dd43f1153d648 into cb632a7926af46c591631a9203c84e5f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:13,158 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:13,158 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/B, priority=13, startTime=1732084933123; duration=0sec 2024-11-20T06:42:13,158 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:13,158 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:B 2024-11-20T06:42:13,158 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:13,159 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:13,159 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 6bac99491880cd23064dd43f1153d648/C is initiating minor compaction (all files) 2024-11-20T06:42:13,159 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6bac99491880cd23064dd43f1153d648/C in TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:13,159 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/348ff2ceebcc4d5e8a8f310fc1f4b7ac, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/8db9662c15b14240b8119091a476e559, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/471a424f0c6d4470876c9cb0d554f8e8] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp, totalSize=36.7 K 2024-11-20T06:42:13,162 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 348ff2ceebcc4d5e8a8f310fc1f4b7ac, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732084930038 2024-11-20T06:42:13,164 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db9662c15b14240b8119091a476e559, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732084930114 2024-11-20T06:42:13,164 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 471a424f0c6d4470876c9cb0d554f8e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732084932276 2024-11-20T06:42:13,176 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6bac99491880cd23064dd43f1153d648#C#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:13,177 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/9ab7694f362d46e98ccd3e8a2a7b30be is 50, key is test_row_0/C:col10/1732084932276/Put/seqid=0 2024-11-20T06:42:13,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742333_1509 (size=13051) 2024-11-20T06:42:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:13,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-20T06:42:13,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/15ecb2ddfe814ef0ad347802e99243eb is 50, key is test_row_0/A:col10/1732084932339/Put/seqid=0 2024-11-20T06:42:13,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742334_1510 (size=14741) 2024-11-20T06:42:13,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084993509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084993510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084993511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084993511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084993514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,546 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/e6901b1c19b34809aba2db8513297903 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e6901b1c19b34809aba2db8513297903 2024-11-20T06:42:13,551 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/A of 6bac99491880cd23064dd43f1153d648 into e6901b1c19b34809aba2db8513297903(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:13,551 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:13,551 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/A, priority=13, startTime=1732084933123; duration=0sec 2024-11-20T06:42:13,551 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:13,551 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:A 2024-11-20T06:42:13,588 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/9ab7694f362d46e98ccd3e8a2a7b30be as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/9ab7694f362d46e98ccd3e8a2a7b30be 2024-11-20T06:42:13,592 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6bac99491880cd23064dd43f1153d648/C of 6bac99491880cd23064dd43f1153d648 into 9ab7694f362d46e98ccd3e8a2a7b30be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:13,592 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:13,592 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., storeName=6bac99491880cd23064dd43f1153d648/C, priority=13, startTime=1732084933123; duration=0sec 2024-11-20T06:42:13,592 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:13,592 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6bac99491880cd23064dd43f1153d648:C 2024-11-20T06:42:13,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084993623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084993623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084993624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084993624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084993624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084993833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084993834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084993834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084993834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:13,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084993834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:13,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/15ecb2ddfe814ef0ad347802e99243eb 2024-11-20T06:42:13,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/ac5a2011012f4657807d3891a881b533 is 50, key is test_row_0/B:col10/1732084932339/Put/seqid=0 2024-11-20T06:42:13,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742335_1511 (size=12301) 2024-11-20T06:42:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T06:42:14,144 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:14,144 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-20T06:42:14,145 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T06:42:14,146 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:14,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:14,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084994154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:14,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084994154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:14,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084994154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:14,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084994154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084994154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T06:42:14,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T06:42:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:14,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:14,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/ac5a2011012f4657807d3891a881b533 2024-11-20T06:42:14,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/114311c2cec945a5ae42b471b636961d is 50, key is test_row_0/C:col10/1732084932339/Put/seqid=0 2024-11-20T06:42:14,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742336_1512 (size=12301) 2024-11-20T06:42:14,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T06:42:14,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:14,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:14,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:14,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:14,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T06:42:14,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T06:42:14,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:14,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. as already flushing 2024-11-20T06:42:14,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:14,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:14,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:14,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59960 deadline: 1732084994664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:14,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59970 deadline: 1732084994664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:14,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59936 deadline: 1732084994664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:14,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59922 deadline: 1732084994664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:14,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59980 deadline: 1732084994664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 , the old value is region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., hostname=61c2066a8a0f,33847,1732084794617, seqNum=2 because the exception is null or not the one we care about 2024-11-20T06:42:14,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/114311c2cec945a5ae42b471b636961d 2024-11-20T06:42:14,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/15ecb2ddfe814ef0ad347802e99243eb as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/15ecb2ddfe814ef0ad347802e99243eb 2024-11-20T06:42:14,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/15ecb2ddfe814ef0ad347802e99243eb, entries=200, sequenceid=363, filesize=14.4 K 2024-11-20T06:42:14,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/ac5a2011012f4657807d3891a881b533 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/ac5a2011012f4657807d3891a881b533 2024-11-20T06:42:14,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/ac5a2011012f4657807d3891a881b533, entries=150, sequenceid=363, filesize=12.0 K 2024-11-20T06:42:14,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/114311c2cec945a5ae42b471b636961d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/114311c2cec945a5ae42b471b636961d 2024-11-20T06:42:14,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/114311c2cec945a5ae42b471b636961d, entries=150, sequenceid=363, filesize=12.0 K 2024-11-20T06:42:14,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6bac99491880cd23064dd43f1153d648 in 1240ms, sequenceid=363, compaction requested=false 2024-11-20T06:42:14,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:14,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T06:42:14,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:14,756 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-20T06:42:14,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/28b060435a0745d0aeaa66db14f56f7a is 50, key is test_row_0/A:col10/1732084933510/Put/seqid=0 2024-11-20T06:42:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T06:42:14,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742337_1513 (size=12301) 2024-11-20T06:42:14,775 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/28b060435a0745d0aeaa66db14f56f7a 2024-11-20T06:42:14,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/50e05cb610a04ce8b9ab188ac7550553 is 50, key is test_row_0/B:col10/1732084933510/Put/seqid=0 2024-11-20T06:42:14,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742338_1514 (size=12301) 2024-11-20T06:42:15,073 INFO [Thread-1956 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1956. 2024-11-20T06:42:15,073 DEBUG [Thread-1956 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:15,073 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,074 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:15,075 INFO [Thread-1950 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1950. 2024-11-20T06:42:15,075 DEBUG [Thread-1950 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:15,075 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,075 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:15,077 INFO [Thread-1952 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1952. 2024-11-20T06:42:15,077 DEBUG [Thread-1952 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:15,077 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,077 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:15,080 INFO [Thread-1948 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1948. 2024-11-20T06:42:15,080 DEBUG [Thread-1948 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:15,080 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,080 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:15,081 INFO [Thread-1954 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1954. 2024-11-20T06:42:15,081 DEBUG [Thread-1954 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:15,081 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,081 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:15,186 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/50e05cb610a04ce8b9ab188ac7550553 2024-11-20T06:42:15,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/628e3a91c92d4c4a846c130fa71648de is 50, key is test_row_0/C:col10/1732084933510/Put/seqid=0 2024-11-20T06:42:15,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742339_1515 (size=12301) 2024-11-20T06:42:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T06:42:15,605 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/628e3a91c92d4c4a846c130fa71648de 2024-11-20T06:42:15,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/28b060435a0745d0aeaa66db14f56f7a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/28b060435a0745d0aeaa66db14f56f7a 2024-11-20T06:42:15,621 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/28b060435a0745d0aeaa66db14f56f7a, entries=150, sequenceid=375, filesize=12.0 K 2024-11-20T06:42:15,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/50e05cb610a04ce8b9ab188ac7550553 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/50e05cb610a04ce8b9ab188ac7550553 2024-11-20T06:42:15,625 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/50e05cb610a04ce8b9ab188ac7550553, entries=150, sequenceid=375, filesize=12.0 K 2024-11-20T06:42:15,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/628e3a91c92d4c4a846c130fa71648de as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/628e3a91c92d4c4a846c130fa71648de 2024-11-20T06:42:15,630 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/628e3a91c92d4c4a846c130fa71648de, entries=150, sequenceid=375, filesize=12.0 K 2024-11-20T06:42:15,631 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 6bac99491880cd23064dd43f1153d648 in 875ms, sequenceid=375, compaction requested=true 2024-11-20T06:42:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 6bac99491880cd23064dd43f1153d648: 2024-11-20T06:42:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-20T06:42:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-20T06:42:15,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-20T06:42:15,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4860 sec 2024-11-20T06:42:15,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.4900 sec 2024-11-20T06:42:15,676 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-11-20T06:42:15,676 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:15,676 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,676 INFO [Thread-1937 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1937. 2024-11-20T06:42:15,676 DEBUG [Thread-1937 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:15,676 DEBUG [Thread-1937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,676 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:15,677 DEBUG [Thread-1937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:15,685 INFO [Thread-1941 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1941. 2024-11-20T06:42:15,685 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-11-20T06:42:15,685 INFO [Thread-1939 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1939. 2024-11-20T06:42:15,685 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:15,685 DEBUG [Thread-1941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:15,685 DEBUG [Thread-1939 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:15,685 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,685 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,685 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,685 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,685 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,685 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:15,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:15,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:15,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T06:42:16,295 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 134 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1652 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4956 rows 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1651 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4953 rows 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1648 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4944 rows 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1647 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4941 rows 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1667 2024-11-20T06:42:16,296 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4995 rows 2024-11-20T06:42:16,297 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:42:16,297 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:42:16,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:16,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:16,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:16,300 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T06:42:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T06:42:16,304 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084936304"}]},"ts":"1732084936304"} 2024-11-20T06:42:16,306 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T06:42:16,306 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T06:42:16,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T06:42:16,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bac99491880cd23064dd43f1153d648, UNASSIGN}] 2024-11-20T06:42:16,310 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bac99491880cd23064dd43f1153d648, UNASSIGN 2024-11-20T06:42:16,311 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=6bac99491880cd23064dd43f1153d648, regionState=CLOSING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:16,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bac99491880cd23064dd43f1153d648, UNASSIGN because future has completed 2024-11-20T06:42:16,313 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T06:42:16,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:42:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T06:42:16,467 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:16,468 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T06:42:16,468 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 6bac99491880cd23064dd43f1153d648, disabling compactions & flushes 2024-11-20T06:42:16,468 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:16,468 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:16,468 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. after waiting 0 ms 2024-11-20T06:42:16,468 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:16,468 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 6bac99491880cd23064dd43f1153d648 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T06:42:16,476 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/21979b0dd19a42dea380791752c083f6 is 50, key is test_row_0/A:col10/1732084935684/Put/seqid=0 2024-11-20T06:42:16,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742340_1516 (size=9857) 2024-11-20T06:42:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T06:42:16,880 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/21979b0dd19a42dea380791752c083f6 2024-11-20T06:42:16,885 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/821f92815cc743589eea4e69acc0da6b is 50, key is test_row_0/B:col10/1732084935684/Put/seqid=0 2024-11-20T06:42:16,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742341_1517 (size=9857) 2024-11-20T06:42:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T06:42:17,289 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/821f92815cc743589eea4e69acc0da6b 2024-11-20T06:42:17,303 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/e8ea66114ced4ac68f37e6bd1bddb41e is 50, key is test_row_0/C:col10/1732084935684/Put/seqid=0 2024-11-20T06:42:17,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742342_1518 (size=9857) 2024-11-20T06:42:17,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T06:42:17,707 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/e8ea66114ced4ac68f37e6bd1bddb41e 2024-11-20T06:42:17,716 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/A/21979b0dd19a42dea380791752c083f6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/21979b0dd19a42dea380791752c083f6 2024-11-20T06:42:17,721 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/21979b0dd19a42dea380791752c083f6, entries=100, sequenceid=383, filesize=9.6 K 2024-11-20T06:42:17,722 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/B/821f92815cc743589eea4e69acc0da6b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/821f92815cc743589eea4e69acc0da6b 2024-11-20T06:42:17,727 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/821f92815cc743589eea4e69acc0da6b, entries=100, sequenceid=383, filesize=9.6 K 2024-11-20T06:42:17,728 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/.tmp/C/e8ea66114ced4ac68f37e6bd1bddb41e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/e8ea66114ced4ac68f37e6bd1bddb41e 2024-11-20T06:42:17,733 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/e8ea66114ced4ac68f37e6bd1bddb41e, entries=100, sequenceid=383, filesize=9.6 K 2024-11-20T06:42:17,734 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 6bac99491880cd23064dd43f1153d648 in 1266ms, sequenceid=383, compaction requested=true 2024-11-20T06:42:17,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/42dd3fcbeb0f4beca7d7bc76991f3107, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e40f36a73eb640f09a47d999ad6d76e3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/edc3e64ea5ec44e89f5c1da6e7eef52e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6935f84549104ae5a5bb9ea2e37164e6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/30ad491ee28240a5b913e8c3a1c1fabd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f4742057272245f3ab7aa4f1ed0aa8a2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/15adc96db91f452c9bc5da8240762326, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e9aa8e55d5444088a7cbb54f232d3fc8, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/024520ad9b1e43669fad54c687d672ea, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/dca3b9d892b44271acc535262fae0ba4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/bcd664ddb13b49d8bc654965e68dc55f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/94990ba96051423cb824555c3454a8e2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/4893feff287d44989fd77e7c865dd166, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f301733f36e74b24ba346f373be71e8e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/ad7f70b35de14c57978c470b425252c7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3d02e6b772134cf994b4129f5e4c1abe, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3247b4e70541428884467d1963f3b51e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6b8edf9f9ab343f48361f2fa9054eec5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/da9646e36bf4413e850255fdcf31c709, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/250882179c3b4903b1f29aaebfc49ce4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/2b2aebae173b4becbcec7a059e346069, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6c5ee5e2cd944f85b8c8db38540a2d04] to archive 2024-11-20T06:42:17,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:42:17,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/42dd3fcbeb0f4beca7d7bc76991f3107 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/42dd3fcbeb0f4beca7d7bc76991f3107 2024-11-20T06:42:17,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e40f36a73eb640f09a47d999ad6d76e3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e40f36a73eb640f09a47d999ad6d76e3 2024-11-20T06:42:17,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/edc3e64ea5ec44e89f5c1da6e7eef52e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/edc3e64ea5ec44e89f5c1da6e7eef52e 2024-11-20T06:42:17,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6935f84549104ae5a5bb9ea2e37164e6 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6935f84549104ae5a5bb9ea2e37164e6 2024-11-20T06:42:17,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/30ad491ee28240a5b913e8c3a1c1fabd to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/30ad491ee28240a5b913e8c3a1c1fabd 2024-11-20T06:42:17,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f4742057272245f3ab7aa4f1ed0aa8a2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f4742057272245f3ab7aa4f1ed0aa8a2 2024-11-20T06:42:17,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/15adc96db91f452c9bc5da8240762326 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/15adc96db91f452c9bc5da8240762326 2024-11-20T06:42:17,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e9aa8e55d5444088a7cbb54f232d3fc8 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e9aa8e55d5444088a7cbb54f232d3fc8 2024-11-20T06:42:17,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/024520ad9b1e43669fad54c687d672ea to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/024520ad9b1e43669fad54c687d672ea 2024-11-20T06:42:17,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/dca3b9d892b44271acc535262fae0ba4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/dca3b9d892b44271acc535262fae0ba4 2024-11-20T06:42:17,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/bcd664ddb13b49d8bc654965e68dc55f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/bcd664ddb13b49d8bc654965e68dc55f 2024-11-20T06:42:17,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/94990ba96051423cb824555c3454a8e2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/94990ba96051423cb824555c3454a8e2 2024-11-20T06:42:17,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/4893feff287d44989fd77e7c865dd166 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/4893feff287d44989fd77e7c865dd166 2024-11-20T06:42:17,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f301733f36e74b24ba346f373be71e8e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/f301733f36e74b24ba346f373be71e8e 2024-11-20T06:42:17,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/ad7f70b35de14c57978c470b425252c7 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/ad7f70b35de14c57978c470b425252c7 2024-11-20T06:42:17,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3d02e6b772134cf994b4129f5e4c1abe to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3d02e6b772134cf994b4129f5e4c1abe 2024-11-20T06:42:17,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3247b4e70541428884467d1963f3b51e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/3247b4e70541428884467d1963f3b51e 2024-11-20T06:42:17,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6b8edf9f9ab343f48361f2fa9054eec5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6b8edf9f9ab343f48361f2fa9054eec5 2024-11-20T06:42:17,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/da9646e36bf4413e850255fdcf31c709 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/da9646e36bf4413e850255fdcf31c709 2024-11-20T06:42:17,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/250882179c3b4903b1f29aaebfc49ce4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/250882179c3b4903b1f29aaebfc49ce4 2024-11-20T06:42:17,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/2b2aebae173b4becbcec7a059e346069 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/2b2aebae173b4becbcec7a059e346069 2024-11-20T06:42:17,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6c5ee5e2cd944f85b8c8db38540a2d04 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/6c5ee5e2cd944f85b8c8db38540a2d04 2024-11-20T06:42:17,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a7c01e5bbe8c4576a2f47c0265d172bc, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/6079a71c390e4b1dba991c333e3ffb1d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/0ce059e1d97346b3a9efb8b10cd072bd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/02c07510238448ea88e86efe664539b2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5f3b06bc562d4aba8b92792ab94d4232, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a65288cdb0674ffc8181394adc719226, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/674282bf068347b6a612a16d21b57d73, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/f71ae55e30ff42d6997b1de88044b1de, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/e3d67f95f98a43528b5bb914d5e046d1, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/2b079ec8e0f9471f985cc1f19e3e3d33, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/014ca312cc7b49b8bd58c59b28e88885, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/b64728d210cf4969a73f62a42c67a69a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/4f1aa34035c148b5bf0b81759f7e365d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/18f49dcf6b0d4a918f40e5f69cc1ce81, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/9c5248f4e2284a8fb0b2eb19878f25c5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/ba9c98b44c3141cf8abc3daf0af2388e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/3a4ffcfae6a5456a877784acc2b66fa5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5a060e2b37764b429e289b1388c87bc0, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/4dc1675c98bb443eb5c659ca03d6eb65, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/245fb7f72c97495b940ecd640ff1f0f1, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/985a6de12db34f6e8d1dd1fa27f3b850, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/cd64fa58b91f403bbee8859eb8c33dbe] to archive 2024-11-20T06:42:17,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:42:17,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a7c01e5bbe8c4576a2f47c0265d172bc to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a7c01e5bbe8c4576a2f47c0265d172bc 2024-11-20T06:42:17,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/6079a71c390e4b1dba991c333e3ffb1d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/6079a71c390e4b1dba991c333e3ffb1d 2024-11-20T06:42:17,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/0ce059e1d97346b3a9efb8b10cd072bd to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/0ce059e1d97346b3a9efb8b10cd072bd 2024-11-20T06:42:17,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/02c07510238448ea88e86efe664539b2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/02c07510238448ea88e86efe664539b2 2024-11-20T06:42:17,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5f3b06bc562d4aba8b92792ab94d4232 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5f3b06bc562d4aba8b92792ab94d4232 2024-11-20T06:42:17,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a65288cdb0674ffc8181394adc719226 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/a65288cdb0674ffc8181394adc719226 2024-11-20T06:42:17,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/674282bf068347b6a612a16d21b57d73 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/674282bf068347b6a612a16d21b57d73 2024-11-20T06:42:17,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/f71ae55e30ff42d6997b1de88044b1de to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/f71ae55e30ff42d6997b1de88044b1de 2024-11-20T06:42:17,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/e3d67f95f98a43528b5bb914d5e046d1 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/e3d67f95f98a43528b5bb914d5e046d1 2024-11-20T06:42:17,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/2b079ec8e0f9471f985cc1f19e3e3d33 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/2b079ec8e0f9471f985cc1f19e3e3d33 2024-11-20T06:42:17,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/014ca312cc7b49b8bd58c59b28e88885 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/014ca312cc7b49b8bd58c59b28e88885 2024-11-20T06:42:17,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/b64728d210cf4969a73f62a42c67a69a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/b64728d210cf4969a73f62a42c67a69a 2024-11-20T06:42:17,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/4f1aa34035c148b5bf0b81759f7e365d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/4f1aa34035c148b5bf0b81759f7e365d 2024-11-20T06:42:17,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/18f49dcf6b0d4a918f40e5f69cc1ce81 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/18f49dcf6b0d4a918f40e5f69cc1ce81 2024-11-20T06:42:17,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/9c5248f4e2284a8fb0b2eb19878f25c5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/9c5248f4e2284a8fb0b2eb19878f25c5 2024-11-20T06:42:17,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/ba9c98b44c3141cf8abc3daf0af2388e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/ba9c98b44c3141cf8abc3daf0af2388e 2024-11-20T06:42:17,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/3a4ffcfae6a5456a877784acc2b66fa5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/3a4ffcfae6a5456a877784acc2b66fa5 2024-11-20T06:42:17,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5a060e2b37764b429e289b1388c87bc0 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/5a060e2b37764b429e289b1388c87bc0 2024-11-20T06:42:17,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/4dc1675c98bb443eb5c659ca03d6eb65 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/4dc1675c98bb443eb5c659ca03d6eb65 2024-11-20T06:42:17,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/245fb7f72c97495b940ecd640ff1f0f1 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/245fb7f72c97495b940ecd640ff1f0f1 2024-11-20T06:42:17,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/985a6de12db34f6e8d1dd1fa27f3b850 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/985a6de12db34f6e8d1dd1fa27f3b850 2024-11-20T06:42:17,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/cd64fa58b91f403bbee8859eb8c33dbe to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/cd64fa58b91f403bbee8859eb8c33dbe 2024-11-20T06:42:17,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/30d98c939e21425788a8a897cfe64da4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6c60e12ba7ce4e7787a88576ce75536d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/240d361f9b794fbcba77089358269c6b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/221b8a1c1b204400ae00ecebf2352c57, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/98a6fe31e0ed403ba427125ca77a2447, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/a8e394010d9e4560b0b36884d484985f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/b0c52afa28924eb0858b8d164c532f86, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1a6a46a815b34a59a80d7d88a702ef21, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c43acc9e3f14468daf710f3d49049fe5, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/04079c997f504c2bbf029bdc1cd3923c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/013ef13044e84f718093bb44d647b889, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/07586265b7c048efb6389b7399d01824, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/271600ce9ae249fb9505f6d51bc8f0b3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/91b9b1d1c5174281a7858c63cb0e63c8, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/5f9e53b9afd641d1bf7cc89a7a034654, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c0552c66914b4318892c3e41eda3a332, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6ecdb66c8e88465c873718ac06d77c00, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1b03fdecd02b4eac930532bddcb0e6f7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/348ff2ceebcc4d5e8a8f310fc1f4b7ac, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/43ac2f747f98494eab9745e727f687bd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/8db9662c15b14240b8119091a476e559, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/471a424f0c6d4470876c9cb0d554f8e8] to archive 2024-11-20T06:42:17,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:42:17,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/30d98c939e21425788a8a897cfe64da4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/30d98c939e21425788a8a897cfe64da4 2024-11-20T06:42:17,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6c60e12ba7ce4e7787a88576ce75536d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6c60e12ba7ce4e7787a88576ce75536d 2024-11-20T06:42:17,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/240d361f9b794fbcba77089358269c6b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/240d361f9b794fbcba77089358269c6b 2024-11-20T06:42:17,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/221b8a1c1b204400ae00ecebf2352c57 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/221b8a1c1b204400ae00ecebf2352c57 2024-11-20T06:42:17,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/98a6fe31e0ed403ba427125ca77a2447 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/98a6fe31e0ed403ba427125ca77a2447 2024-11-20T06:42:17,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/a8e394010d9e4560b0b36884d484985f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/a8e394010d9e4560b0b36884d484985f 2024-11-20T06:42:17,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/b0c52afa28924eb0858b8d164c532f86 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/b0c52afa28924eb0858b8d164c532f86 2024-11-20T06:42:17,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1a6a46a815b34a59a80d7d88a702ef21 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1a6a46a815b34a59a80d7d88a702ef21 2024-11-20T06:42:17,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c43acc9e3f14468daf710f3d49049fe5 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c43acc9e3f14468daf710f3d49049fe5 2024-11-20T06:42:17,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/04079c997f504c2bbf029bdc1cd3923c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/04079c997f504c2bbf029bdc1cd3923c 2024-11-20T06:42:17,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/013ef13044e84f718093bb44d647b889 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/013ef13044e84f718093bb44d647b889 2024-11-20T06:42:17,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/07586265b7c048efb6389b7399d01824 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/07586265b7c048efb6389b7399d01824 2024-11-20T06:42:17,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/271600ce9ae249fb9505f6d51bc8f0b3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/271600ce9ae249fb9505f6d51bc8f0b3 2024-11-20T06:42:17,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/91b9b1d1c5174281a7858c63cb0e63c8 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/91b9b1d1c5174281a7858c63cb0e63c8 2024-11-20T06:42:17,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/5f9e53b9afd641d1bf7cc89a7a034654 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/5f9e53b9afd641d1bf7cc89a7a034654 2024-11-20T06:42:17,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c0552c66914b4318892c3e41eda3a332 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/c0552c66914b4318892c3e41eda3a332 2024-11-20T06:42:17,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6ecdb66c8e88465c873718ac06d77c00 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/6ecdb66c8e88465c873718ac06d77c00 2024-11-20T06:42:17,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1b03fdecd02b4eac930532bddcb0e6f7 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/1b03fdecd02b4eac930532bddcb0e6f7 2024-11-20T06:42:17,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/348ff2ceebcc4d5e8a8f310fc1f4b7ac to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/348ff2ceebcc4d5e8a8f310fc1f4b7ac 2024-11-20T06:42:17,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/43ac2f747f98494eab9745e727f687bd to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/43ac2f747f98494eab9745e727f687bd 2024-11-20T06:42:17,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/8db9662c15b14240b8119091a476e559 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/8db9662c15b14240b8119091a476e559 2024-11-20T06:42:17,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/471a424f0c6d4470876c9cb0d554f8e8 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/471a424f0c6d4470876c9cb0d554f8e8 2024-11-20T06:42:17,791 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/recovered.edits/386.seqid, newMaxSeqId=386, maxSeqId=1 2024-11-20T06:42:17,794 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. 2024-11-20T06:42:17,794 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 6bac99491880cd23064dd43f1153d648: Waiting for close lock at 1732084936468Running coprocessor pre-close hooks at 1732084936468Disabling compacts and flushes for region at 1732084936468Disabling writes for close at 1732084936468Obtaining lock to block concurrent updates at 1732084936468Preparing flush snapshotting stores in 6bac99491880cd23064dd43f1153d648 at 1732084936468Finished memstore snapshotting TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732084936469 (+1 ms)Flushing stores of TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648. at 1732084936471 (+2 ms)Flushing 6bac99491880cd23064dd43f1153d648/A: creating writer at 1732084936471Flushing 6bac99491880cd23064dd43f1153d648/A: appending metadata at 1732084936475 (+4 ms)Flushing 6bac99491880cd23064dd43f1153d648/A: closing flushed file at 1732084936475Flushing 6bac99491880cd23064dd43f1153d648/B: creating writer at 1732084936883 (+408 ms)Flushing 6bac99491880cd23064dd43f1153d648/B: appending metadata at 1732084936885 (+2 ms)Flushing 6bac99491880cd23064dd43f1153d648/B: closing flushed file at 1732084936885Flushing 6bac99491880cd23064dd43f1153d648/C: creating writer at 1732084937298 (+413 ms)Flushing 6bac99491880cd23064dd43f1153d648/C: appending metadata at 1732084937302 (+4 ms)Flushing 6bac99491880cd23064dd43f1153d648/C: closing flushed file at 1732084937302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4de92606: reopening flushed file at 1732084937714 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@187c191: reopening flushed file at 1732084937721 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c88ed70: reopening flushed file at 1732084937728 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 6bac99491880cd23064dd43f1153d648 in 1266ms, sequenceid=383, compaction requested=true at 1732084937734 (+6 ms)Writing region close event to WAL at 1732084937788 (+54 ms)Running coprocessor post-close hooks at 1732084937793 (+5 ms)Closed at 1732084937793 2024-11-20T06:42:17,795 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:17,796 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=6bac99491880cd23064dd43f1153d648, regionState=CLOSED 2024-11-20T06:42:17,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:42:17,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-20T06:42:17,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 6bac99491880cd23064dd43f1153d648, server=61c2066a8a0f,33847,1732084794617 in 1.4840 sec 2024-11-20T06:42:17,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-20T06:42:17,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6bac99491880cd23064dd43f1153d648, UNASSIGN in 1.4900 sec 2024-11-20T06:42:17,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-20T06:42:17,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4940 sec 2024-11-20T06:42:17,803 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084937803"}]},"ts":"1732084937803"} 2024-11-20T06:42:17,804 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T06:42:17,804 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T06:42:17,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5040 sec 2024-11-20T06:42:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T06:42:18,456 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:18,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T06:42:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:18,462 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T06:42:18,464 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:18,468 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:18,471 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/recovered.edits] 2024-11-20T06:42:18,475 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/15ecb2ddfe814ef0ad347802e99243eb to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/15ecb2ddfe814ef0ad347802e99243eb 2024-11-20T06:42:18,477 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/21979b0dd19a42dea380791752c083f6 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/21979b0dd19a42dea380791752c083f6 2024-11-20T06:42:18,479 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/28b060435a0745d0aeaa66db14f56f7a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/28b060435a0745d0aeaa66db14f56f7a 2024-11-20T06:42:18,481 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e6901b1c19b34809aba2db8513297903 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/A/e6901b1c19b34809aba2db8513297903 2024-11-20T06:42:18,484 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/50e05cb610a04ce8b9ab188ac7550553 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/50e05cb610a04ce8b9ab188ac7550553 2024-11-20T06:42:18,486 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/821f92815cc743589eea4e69acc0da6b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/821f92815cc743589eea4e69acc0da6b 2024-11-20T06:42:18,487 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/ac5a2011012f4657807d3891a881b533 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/ac5a2011012f4657807d3891a881b533 2024-11-20T06:42:18,488 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/cb632a7926af46c591631a9203c84e5f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/B/cb632a7926af46c591631a9203c84e5f 2024-11-20T06:42:18,490 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/114311c2cec945a5ae42b471b636961d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/114311c2cec945a5ae42b471b636961d 2024-11-20T06:42:18,491 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/628e3a91c92d4c4a846c130fa71648de to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/628e3a91c92d4c4a846c130fa71648de 2024-11-20T06:42:18,492 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/9ab7694f362d46e98ccd3e8a2a7b30be to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/9ab7694f362d46e98ccd3e8a2a7b30be 2024-11-20T06:42:18,493 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/e8ea66114ced4ac68f37e6bd1bddb41e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/C/e8ea66114ced4ac68f37e6bd1bddb41e 2024-11-20T06:42:18,494 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/recovered.edits/386.seqid to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648/recovered.edits/386.seqid 2024-11-20T06:42:18,495 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/6bac99491880cd23064dd43f1153d648 2024-11-20T06:42:18,495 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T06:42:18,497 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:18,498 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T06:42:18,500 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T06:42:18,501 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:18,501 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T06:42:18,501 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732084938501"}]},"ts":"9223372036854775807"} 2024-11-20T06:42:18,503 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T06:42:18,503 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6bac99491880cd23064dd43f1153d648, NAME => 'TestAcidGuarantees,,1732084912818.6bac99491880cd23064dd43f1153d648.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T06:42:18,503 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T06:42:18,503 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732084938503"}]},"ts":"9223372036854775807"} 2024-11-20T06:42:18,505 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T06:42:18,506 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:18,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-20T06:42:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T06:42:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T06:42:18,574 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:18,584 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 240), OpenFileDescriptor=447 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=361 (was 404), ProcessCount=11 (was 11), AvailableMemoryMB=4403 (was 4439) 2024-11-20T06:42:18,596 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=361, ProcessCount=11, AvailableMemoryMB=4403 2024-11-20T06:42:18,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T06:42:18,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:42:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:18,599 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T06:42:18,600 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:18,600 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-20T06:42:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T06:42:18,601 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T06:42:18,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742343_1519 (size=959) 2024-11-20T06:42:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T06:42:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T06:42:19,012 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5 2024-11-20T06:42:19,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742344_1520 (size=53) 2024-11-20T06:42:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T06:42:19,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:42:19,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 03bb109fa60f64b2fccb76b2907dd474, disabling compactions & flushes 2024-11-20T06:42:19,425 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:19,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:19,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. after waiting 0 ms 2024-11-20T06:42:19,425 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:19,425 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:19,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 03bb109fa60f64b2fccb76b2907dd474: Waiting for close lock at 1732084939425Disabling compacts and flushes for region at 1732084939425Disabling writes for close at 1732084939425Writing region close event to WAL at 1732084939425Closed at 1732084939425 2024-11-20T06:42:19,429 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T06:42:19,429 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732084939429"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732084939429"}]},"ts":"1732084939429"} 2024-11-20T06:42:19,433 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T06:42:19,435 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T06:42:19,435 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084939435"}]},"ts":"1732084939435"} 2024-11-20T06:42:19,436 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T06:42:19,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, ASSIGN}] 2024-11-20T06:42:19,438 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, ASSIGN 2024-11-20T06:42:19,438 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, ASSIGN; state=OFFLINE, location=61c2066a8a0f,33847,1732084794617; forceNewPlan=false, retain=false 2024-11-20T06:42:19,589 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=03bb109fa60f64b2fccb76b2907dd474, regionState=OPENING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:19,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, ASSIGN because future has completed 2024-11-20T06:42:19,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:42:19,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T06:42:19,752 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:19,752 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:42:19,752 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,753 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:42:19,753 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,753 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,754 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,756 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03bb109fa60f64b2fccb76b2907dd474 columnFamilyName A 2024-11-20T06:42:19,756 DEBUG [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:19,757 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(327): Store=03bb109fa60f64b2fccb76b2907dd474/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:42:19,757 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,759 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03bb109fa60f64b2fccb76b2907dd474 columnFamilyName B 2024-11-20T06:42:19,759 DEBUG [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:19,760 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(327): Store=03bb109fa60f64b2fccb76b2907dd474/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:42:19,760 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,761 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03bb109fa60f64b2fccb76b2907dd474 columnFamilyName C 2024-11-20T06:42:19,761 DEBUG [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:19,762 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(327): Store=03bb109fa60f64b2fccb76b2907dd474/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:42:19,762 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,763 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,764 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,766 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,766 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,767 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:42:19,769 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,773 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:42:19,774 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 03bb109fa60f64b2fccb76b2907dd474; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71096943, jitterRate=0.05942700803279877}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:42:19,774 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:19,774 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 03bb109fa60f64b2fccb76b2907dd474: Running coprocessor pre-open hook at 1732084939753Writing region info on filesystem at 1732084939753Initializing all the Stores at 1732084939754 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084939754Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084939754Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084939754Cleaning up temporary data from old regions at 1732084939766 (+12 ms)Running coprocessor post-open hooks at 1732084939774 (+8 ms)Region opened successfully at 1732084939774 2024-11-20T06:42:19,775 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., pid=144, masterSystemTime=1732084939745 2024-11-20T06:42:19,777 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:19,777 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:19,777 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=03bb109fa60f64b2fccb76b2907dd474, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:19,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:42:19,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-20T06:42:19,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 in 188 msec 2024-11-20T06:42:19,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-20T06:42:19,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, ASSIGN in 344 msec 2024-11-20T06:42:19,784 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T06:42:19,784 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084939784"}]},"ts":"1732084939784"} 2024-11-20T06:42:19,785 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T06:42:19,786 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T06:42:19,788 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1890 sec 2024-11-20T06:42:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-20T06:42:20,745 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:20,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b3b1fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:20,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:20,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:20,749 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c799b60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:20,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@271581fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:20,753 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:20,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:20,756 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T06:42:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,33659,1732084793752 2024-11-20T06:42:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44b7fd24 2024-11-20T06:42:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T06:42:20,758 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T06:42:20,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T06:42:20,759 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:42:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:20,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742345_1521 (size=995) 2024-11-20T06:42:21,170 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-20T06:42:21,170 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-20T06:42:21,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T06:42:21,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, REOPEN/MOVE}] 2024-11-20T06:42:21,178 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, REOPEN/MOVE 2024-11-20T06:42:21,179 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=03bb109fa60f64b2fccb76b2907dd474, regionState=CLOSING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, REOPEN/MOVE because future has completed 2024-11-20T06:42:21,181 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T06:42:21,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:42:21,334 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,334 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T06:42:21,334 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 03bb109fa60f64b2fccb76b2907dd474, disabling compactions & flushes 2024-11-20T06:42:21,334 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:21,334 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:21,334 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. after waiting 0 ms 2024-11-20T06:42:21,334 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:21,343 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T06:42:21,344 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:21,344 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 03bb109fa60f64b2fccb76b2907dd474: Waiting for close lock at 1732084941334Running coprocessor pre-close hooks at 1732084941334Disabling compacts and flushes for region at 1732084941334Disabling writes for close at 1732084941334Writing region close event to WAL at 1732084941335 (+1 ms)Running coprocessor post-close hooks at 1732084941344 (+9 ms)Closed at 1732084941344 2024-11-20T06:42:21,344 WARN [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 03bb109fa60f64b2fccb76b2907dd474 to self. 2024-11-20T06:42:21,345 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=03bb109fa60f64b2fccb76b2907dd474, regionState=CLOSED 2024-11-20T06:42:21,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:42:21,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-20T06:42:21,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 in 167 msec 2024-11-20T06:42:21,350 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, REOPEN/MOVE; state=CLOSED, location=61c2066a8a0f,33847,1732084794617; forceNewPlan=false, retain=true 2024-11-20T06:42:21,501 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=03bb109fa60f64b2fccb76b2907dd474, regionState=OPENING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, REOPEN/MOVE because future has completed 2024-11-20T06:42:21,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:42:21,667 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:21,668 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:42:21,669 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,669 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:42:21,669 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,670 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,671 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,672 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03bb109fa60f64b2fccb76b2907dd474 columnFamilyName A 2024-11-20T06:42:21,673 DEBUG [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:21,674 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(327): Store=03bb109fa60f64b2fccb76b2907dd474/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:42:21,674 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,675 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03bb109fa60f64b2fccb76b2907dd474 columnFamilyName B 2024-11-20T06:42:21,675 DEBUG [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:21,676 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(327): Store=03bb109fa60f64b2fccb76b2907dd474/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:42:21,676 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,676 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03bb109fa60f64b2fccb76b2907dd474 columnFamilyName C 2024-11-20T06:42:21,676 DEBUG [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:21,677 INFO [StoreOpener-03bb109fa60f64b2fccb76b2907dd474-1 {}] regionserver.HStore(327): Store=03bb109fa60f64b2fccb76b2907dd474/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:42:21,677 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,678 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,679 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,680 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,680 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,681 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T06:42:21,682 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,683 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 03bb109fa60f64b2fccb76b2907dd474; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72771636, jitterRate=0.08438187837600708}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T06:42:21,683 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,684 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 03bb109fa60f64b2fccb76b2907dd474: Running coprocessor pre-open hook at 1732084941670Writing region info on filesystem at 1732084941670Initializing all the Stores at 1732084941671 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084941671Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084941671Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732084941671Cleaning up temporary data from old regions at 1732084941680 (+9 ms)Running coprocessor post-open hooks at 1732084941683 (+3 ms)Region opened successfully at 1732084941684 (+1 ms) 2024-11-20T06:42:21,685 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., pid=149, masterSystemTime=1732084941660 2024-11-20T06:42:21,686 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:21,686 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:21,687 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=03bb109fa60f64b2fccb76b2907dd474, regionState=OPEN, openSeqNum=5, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:42:21,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-20T06:42:21,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 in 184 msec 2024-11-20T06:42:21,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-20T06:42:21,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, REOPEN/MOVE in 513 msec 2024-11-20T06:42:21,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-20T06:42:21,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 519 msec 2024-11-20T06:42:21,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 934 msec 2024-11-20T06:42:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T06:42:21,696 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:21,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22842932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:21,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:21,697 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:21,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:21,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87ce4f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:21,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:21,698 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@264fb1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5fa63a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:21,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:21,700 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217aaa97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:21,701 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f9a414, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a933d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:21,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:21,703 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bf139e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:21,704 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d101a52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f73499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:21,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:21,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f98cfd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:21,707 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a33d14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f42478c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:21,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:21,708 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@419cab8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:21,709 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36fe69a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6185aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:21,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:21,711 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd8fd43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:21,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:21,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce801d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476ca966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:21,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:21,713 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4897124a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:21,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:21,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d266586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b91db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:21,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:21,717 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd8bb69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:21,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:21,718 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:21,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f65974, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dcf631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:21,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:21,720 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:21,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:21,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@198e61f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:21,721 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49051295, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3d9872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,33659,-1 for getting cluster id 2024-11-20T06:42:21,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:42:21,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f056ae1d-0bc3-461f-a70b-1784272b5485' 2024-11-20T06:42:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:42:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f056ae1d-0bc3-461f-a70b-1784272b5485" 2024-11-20T06:42:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec03df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,33659,-1] 2024-11-20T06:42:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:42:21,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:21,725 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:42:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c37ef9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:42:21,726 DEBUG [Thread-2333 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:21,726 DEBUG [Thread-2336 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:21,726 DEBUG [Thread-2329 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:21,726 DEBUG [Thread-2342 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:21,726 DEBUG [Thread-2327 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:21,727 DEBUG [Thread-2338 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:21,727 DEBUG [Thread-2331 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:21,728 DEBUG [Thread-2338 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:21,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:21,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:21,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:21,728 DEBUG [Thread-2338 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:21,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:21,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:21,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:21,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:21,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:21,729 DEBUG [Thread-2340 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:21,729 DEBUG [Thread-2325 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:21,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:42:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:42:21,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:42:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-20T06:42:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:42:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:42:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:42:21,732 DEBUG [Thread-2344 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:42:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:42:21,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T06:42:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,33847,1732084794617, seqNum=-1] 2024-11-20T06:42:21,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:42:21,734 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:42:21,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:42:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:42:21,736 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:21,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5] 2024-11-20T06:42:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,740 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:21,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-20T06:42:21,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209e31991e6f134ccbae94714c93964c15_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084941739/Put/seqid=0 2024-11-20T06:42:21,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085001766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:21,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085001767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:21,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085001767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:21,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:21,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085001770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085001772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:21,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742346_1522 (size=12154) 2024-11-20T06:42:21,780 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:21,783 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209e31991e6f134ccbae94714c93964c15_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e31991e6f134ccbae94714c93964c15_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:21,784 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/5ad44c8195cf43e78872ebb3cf82141b, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:21,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/5ad44c8195cf43e78872ebb3cf82141b is 175, key is test_row_0/A:col10/1732084941739/Put/seqid=0 2024-11-20T06:42:21,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742347_1523 (size=30955) 2024-11-20T06:42:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T06:42:21,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085001873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085001873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:21,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085001884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:21,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085001884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:21,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085001884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:21,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:21,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T06:42:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:21,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T06:42:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T06:42:22,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085002083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085002084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085002094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085002094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085002094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T06:42:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,230 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/5ad44c8195cf43e78872ebb3cf82141b 2024-11-20T06:42:22,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/4751771786d0483fa49df93f5d085acc is 50, key is test_row_0/B:col10/1732084941739/Put/seqid=0 2024-11-20T06:42:22,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742348_1524 (size=12001) 2024-11-20T06:42:22,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/4751771786d0483fa49df93f5d085acc 2024-11-20T06:42:22,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/0d22a543f9d94dc6a2444ae801fa4bc1 is 50, key is test_row_0/C:col10/1732084941739/Put/seqid=0 2024-11-20T06:42:22,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742349_1525 (size=12001) 2024-11-20T06:42:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T06:42:22,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T06:42:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085002393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085002393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085002404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085002404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085002404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T06:42:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/0d22a543f9d94dc6a2444ae801fa4bc1 2024-11-20T06:42:22,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/5ad44c8195cf43e78872ebb3cf82141b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/5ad44c8195cf43e78872ebb3cf82141b 2024-11-20T06:42:22,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T06:42:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/5ad44c8195cf43e78872ebb3cf82141b, entries=150, sequenceid=15, filesize=30.2 K 2024-11-20T06:42:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/4751771786d0483fa49df93f5d085acc as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/4751771786d0483fa49df93f5d085acc 2024-11-20T06:42:22,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/4751771786d0483fa49df93f5d085acc, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T06:42:22,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/0d22a543f9d94dc6a2444ae801fa4bc1 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/0d22a543f9d94dc6a2444ae801fa4bc1 2024-11-20T06:42:22,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/0d22a543f9d94dc6a2444ae801fa4bc1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T06:42:22,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 03bb109fa60f64b2fccb76b2907dd474 in 956ms, sequenceid=15, compaction requested=false 2024-11-20T06:42:22,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:22,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-20T06:42:22,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:22,841 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-20T06:42:22,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201d8d0743c7604713b60aed330357f949_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084941764/Put/seqid=0 2024-11-20T06:42:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T06:42:22,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742350_1526 (size=12154) 2024-11-20T06:42:22,879 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T06:42:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:22,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085002914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085002915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085002918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085002919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:22,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085002920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085003024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085003033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085003033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085003034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085003234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085003243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085003243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085003243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:23,280 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201d8d0743c7604713b60aed330357f949_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201d8d0743c7604713b60aed330357f949_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:23,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/23a3e1b62d494ef9aff5ab10bca6c62d, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:23,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/23a3e1b62d494ef9aff5ab10bca6c62d is 175, key is test_row_0/A:col10/1732084941764/Put/seqid=0 2024-11-20T06:42:23,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742351_1527 (size=30955) 2024-11-20T06:42:23,299 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/23a3e1b62d494ef9aff5ab10bca6c62d 2024-11-20T06:42:23,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/394a6ef5621a4c729f63756eaba98653 is 50, key is test_row_0/B:col10/1732084941764/Put/seqid=0 2024-11-20T06:42:23,317 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T06:42:23,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742352_1528 (size=12001) 2024-11-20T06:42:23,346 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/394a6ef5621a4c729f63756eaba98653 2024-11-20T06:42:23,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/90cb0fa3d0584ba1b3878c83cd297f07 is 50, key is test_row_0/C:col10/1732084941764/Put/seqid=0 2024-11-20T06:42:23,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742353_1529 (size=12001) 2024-11-20T06:42:23,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085003544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085003554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085003554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085003554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:23,765 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/90cb0fa3d0584ba1b3878c83cd297f07 2024-11-20T06:42:23,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/23a3e1b62d494ef9aff5ab10bca6c62d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/23a3e1b62d494ef9aff5ab10bca6c62d 2024-11-20T06:42:23,771 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/23a3e1b62d494ef9aff5ab10bca6c62d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-20T06:42:23,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/394a6ef5621a4c729f63756eaba98653 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/394a6ef5621a4c729f63756eaba98653 2024-11-20T06:42:23,774 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/394a6ef5621a4c729f63756eaba98653, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T06:42:23,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/90cb0fa3d0584ba1b3878c83cd297f07 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/90cb0fa3d0584ba1b3878c83cd297f07 2024-11-20T06:42:23,777 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/90cb0fa3d0584ba1b3878c83cd297f07, entries=150, sequenceid=40, filesize=11.7 K 2024-11-20T06:42:23,778 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 03bb109fa60f64b2fccb76b2907dd474 in 937ms, sequenceid=40, compaction requested=false 2024-11-20T06:42:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:23,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-20T06:42:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-20T06:42:23,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-20T06:42:23,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0470 sec 2024-11-20T06:42:23,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.0580 sec 2024-11-20T06:42:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-20T06:42:23,884 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:23,884 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-20T06:42:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T06:42:23,886 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:23,887 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:23,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:23,926 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:23,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-20T06:42:23,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202b3a9e94d0bc4c8298b147b67f8d4d06_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084942918/Put/seqid=0 2024-11-20T06:42:23,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742354_1530 (size=14594) 2024-11-20T06:42:23,945 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:23,950 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202b3a9e94d0bc4c8298b147b67f8d4d06_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202b3a9e94d0bc4c8298b147b67f8d4d06_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:23,951 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/cb9fe7d029284f868de3a910e92c2df7, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:23,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/cb9fe7d029284f868de3a910e92c2df7 is 175, key is test_row_0/A:col10/1732084942918/Put/seqid=0 2024-11-20T06:42:23,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742355_1531 (size=39549) 2024-11-20T06:42:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T06:42:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085004019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:24,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T06:42:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085004054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:24,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085004064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:24,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085004064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:24,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085004064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:24,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085004134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:24,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T06:42:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T06:42:24,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085004343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:24,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T06:42:24,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:24,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,384 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/cb9fe7d029284f868de3a910e92c2df7 2024-11-20T06:42:24,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/d6409f137c94482c8d405497e7021e77 is 50, key is test_row_0/B:col10/1732084942918/Put/seqid=0 2024-11-20T06:42:24,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742356_1532 (size=12001) 2024-11-20T06:42:24,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T06:42:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T06:42:24,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T06:42:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085004654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:24,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:24,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/d6409f137c94482c8d405497e7021e77 2024-11-20T06:42:24,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a2f7ed59fe4a47bbb153b1299ae9ac11 is 50, key is test_row_0/C:col10/1732084942918/Put/seqid=0 2024-11-20T06:42:24,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T06:42:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:24,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742357_1533 (size=12001) 2024-11-20T06:42:24,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a2f7ed59fe4a47bbb153b1299ae9ac11 2024-11-20T06:42:24,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/cb9fe7d029284f868de3a910e92c2df7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/cb9fe7d029284f868de3a910e92c2df7 2024-11-20T06:42:24,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/cb9fe7d029284f868de3a910e92c2df7, entries=200, sequenceid=53, filesize=38.6 K 2024-11-20T06:42:24,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/d6409f137c94482c8d405497e7021e77 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d6409f137c94482c8d405497e7021e77 2024-11-20T06:42:24,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d6409f137c94482c8d405497e7021e77, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T06:42:24,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a2f7ed59fe4a47bbb153b1299ae9ac11 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f7ed59fe4a47bbb153b1299ae9ac11 2024-11-20T06:42:24,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f7ed59fe4a47bbb153b1299ae9ac11, entries=150, sequenceid=53, filesize=11.7 K 2024-11-20T06:42:24,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 03bb109fa60f64b2fccb76b2907dd474 in 894ms, sequenceid=53, compaction requested=true 2024-11-20T06:42:24,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:24,821 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:24,821 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:24,821 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:24,821 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:24,821 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/B is initiating minor compaction (all files) 2024-11-20T06:42:24,821 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/A is initiating minor compaction (all files) 2024-11-20T06:42:24,821 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/B in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,821 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/A in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,821 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/4751771786d0483fa49df93f5d085acc, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/394a6ef5621a4c729f63756eaba98653, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d6409f137c94482c8d405497e7021e77] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=35.2 K 2024-11-20T06:42:24,821 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/5ad44c8195cf43e78872ebb3cf82141b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/23a3e1b62d494ef9aff5ab10bca6c62d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/cb9fe7d029284f868de3a910e92c2df7] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=99.1 K 2024-11-20T06:42:24,822 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,822 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/5ad44c8195cf43e78872ebb3cf82141b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/23a3e1b62d494ef9aff5ab10bca6c62d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/cb9fe7d029284f868de3a910e92c2df7] 2024-11-20T06:42:24,822 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 4751771786d0483fa49df93f5d085acc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732084941734 2024-11-20T06:42:24,822 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad44c8195cf43e78872ebb3cf82141b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732084941734 2024-11-20T06:42:24,822 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 394a6ef5621a4c729f63756eaba98653, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732084941764 2024-11-20T06:42:24,822 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d6409f137c94482c8d405497e7021e77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732084942918 2024-11-20T06:42:24,822 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23a3e1b62d494ef9aff5ab10bca6c62d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732084941764 2024-11-20T06:42:24,823 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb9fe7d029284f868de3a910e92c2df7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732084942906 2024-11-20T06:42:24,827 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:24,828 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120309e62a373d6461ab4bf8a0c3042909d_03bb109fa60f64b2fccb76b2907dd474 store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:24,829 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:24,829 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/c1eea16ce2244daebf64ffe32cbe9736 is 50, key is test_row_0/B:col10/1732084942918/Put/seqid=0 2024-11-20T06:42:24,830 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120309e62a373d6461ab4bf8a0c3042909d_03bb109fa60f64b2fccb76b2907dd474, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:24,830 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120309e62a373d6461ab4bf8a0c3042909d_03bb109fa60f64b2fccb76b2907dd474 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:24,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742359_1535 (size=4469) 2024-11-20T06:42:24,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742358_1534 (size=12104) 2024-11-20T06:42:24,839 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/c1eea16ce2244daebf64ffe32cbe9736 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/c1eea16ce2244daebf64ffe32cbe9736 2024-11-20T06:42:24,843 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/B of 03bb109fa60f64b2fccb76b2907dd474 into c1eea16ce2244daebf64ffe32cbe9736(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:24,843 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:24,843 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/B, priority=13, startTime=1732084944821; duration=0sec 2024-11-20T06:42:24,843 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:24,843 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:B 2024-11-20T06:42:24,843 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:24,844 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:24,844 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/C is initiating minor compaction (all files) 2024-11-20T06:42:24,844 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/C in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,844 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/0d22a543f9d94dc6a2444ae801fa4bc1, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/90cb0fa3d0584ba1b3878c83cd297f07, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f7ed59fe4a47bbb153b1299ae9ac11] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=35.2 K 2024-11-20T06:42:24,844 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d22a543f9d94dc6a2444ae801fa4bc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732084941734 2024-11-20T06:42:24,845 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 90cb0fa3d0584ba1b3878c83cd297f07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732084941764 2024-11-20T06:42:24,845 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f7ed59fe4a47bbb153b1299ae9ac11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732084942918 2024-11-20T06:42:24,852 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#C#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:24,852 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/bb166f21124644c1b61d5925f2d6f246 is 50, key is test_row_0/C:col10/1732084942918/Put/seqid=0 2024-11-20T06:42:24,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742360_1536 (size=12104) 2024-11-20T06:42:24,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T06:42:24,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:24,956 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-20T06:42:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201126b8def5be40759602890e9912fd7e_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084944017/Put/seqid=0 2024-11-20T06:42:24,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742361_1537 (size=12154) 2024-11-20T06:42:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T06:42:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:25,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:25,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085005082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085005083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085005083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,088 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085005088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085005168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085005194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085005194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085005204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,234 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#A#compaction#462 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:25,235 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/945f8965d015479ab454d5c8500b75ff is 175, key is test_row_0/A:col10/1732084942918/Put/seqid=0 2024-11-20T06:42:25,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742362_1538 (size=31058) 2024-11-20T06:42:25,244 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/945f8965d015479ab454d5c8500b75ff as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/945f8965d015479ab454d5c8500b75ff 2024-11-20T06:42:25,247 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/A of 03bb109fa60f64b2fccb76b2907dd474 into 945f8965d015479ab454d5c8500b75ff(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:25,247 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:25,247 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/A, priority=13, startTime=1732084944821; duration=0sec 2024-11-20T06:42:25,247 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:25,247 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:A 2024-11-20T06:42:25,260 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/bb166f21124644c1b61d5925f2d6f246 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/bb166f21124644c1b61d5925f2d6f246 2024-11-20T06:42:25,263 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/C of 03bb109fa60f64b2fccb76b2907dd474 into bb166f21124644c1b61d5925f2d6f246(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:25,263 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:25,263 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/C, priority=13, startTime=1732084944821; duration=0sec 2024-11-20T06:42:25,264 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:25,264 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:C 2024-11-20T06:42:25,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:25,369 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201126b8def5be40759602890e9912fd7e_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201126b8def5be40759602890e9912fd7e_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:25,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/77f924a86f55465ab7f0023c8e566065, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:25,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/77f924a86f55465ab7f0023c8e566065 is 175, key is test_row_0/A:col10/1732084944017/Put/seqid=0 2024-11-20T06:42:25,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742363_1539 (size=30955) 2024-11-20T06:42:25,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085005404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085005404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085005414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085005714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085005714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085005723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:25,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:25,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:25,775 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/77f924a86f55465ab7f0023c8e566065 2024-11-20T06:42:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/27d6f1f0b6214f229e3f244f01498248 is 50, key is test_row_0/B:col10/1732084944017/Put/seqid=0 2024-11-20T06:42:25,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742364_1540 (size=12001) 2024-11-20T06:42:25,793 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/27d6f1f0b6214f229e3f244f01498248 2024-11-20T06:42:25,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/b23878f2c3b94a0cbf36b78ff1d6a858 is 50, key is test_row_0/C:col10/1732084944017/Put/seqid=0 2024-11-20T06:42:25,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742365_1541 (size=12001) 2024-11-20T06:42:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T06:42:26,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085006184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,186 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,208 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/b23878f2c3b94a0cbf36b78ff1d6a858 2024-11-20T06:42:26,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/77f924a86f55465ab7f0023c8e566065 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/77f924a86f55465ab7f0023c8e566065 2024-11-20T06:42:26,214 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/77f924a86f55465ab7f0023c8e566065, entries=150, sequenceid=77, filesize=30.2 K 2024-11-20T06:42:26,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/27d6f1f0b6214f229e3f244f01498248 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/27d6f1f0b6214f229e3f244f01498248 2024-11-20T06:42:26,217 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/27d6f1f0b6214f229e3f244f01498248, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T06:42:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/b23878f2c3b94a0cbf36b78ff1d6a858 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b23878f2c3b94a0cbf36b78ff1d6a858 2024-11-20T06:42:26,220 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b23878f2c3b94a0cbf36b78ff1d6a858, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T06:42:26,221 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 03bb109fa60f64b2fccb76b2907dd474 in 1265ms, sequenceid=77, compaction requested=false 2024-11-20T06:42:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-20T06:42:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-20T06:42:26,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-20T06:42:26,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3360 sec 2024-11-20T06:42:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:26,226 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:26,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.3410 sec 2024-11-20T06:42:26,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T06:42:26,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c06a6c72b51141678df7a3b4713a204e_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084945082/Put/seqid=0 2024-11-20T06:42:26,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742366_1542 (size=14594) 2024-11-20T06:42:26,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:26,237 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c06a6c72b51141678df7a3b4713a204e_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c06a6c72b51141678df7a3b4713a204e_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:26,238 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bffc3eabcbf947b08b5f791dfd2e9129, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:26,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bffc3eabcbf947b08b5f791dfd2e9129 is 175, key is test_row_0/A:col10/1732084945082/Put/seqid=0 2024-11-20T06:42:26,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742367_1543 (size=39549) 2024-11-20T06:42:26,242 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bffc3eabcbf947b08b5f791dfd2e9129 2024-11-20T06:42:26,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/97537c9c02f3417db4bf4f47c1650136 is 50, key is test_row_0/B:col10/1732084945082/Put/seqid=0 2024-11-20T06:42:26,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742368_1544 (size=12001) 2024-11-20T06:42:26,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085006256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085006258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085006259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085006364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085006364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085006374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085006574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085006574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085006584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/97537c9c02f3417db4bf4f47c1650136 2024-11-20T06:42:26,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/4495ccab2ffe45caa464b91e867f3089 is 50, key is test_row_0/C:col10/1732084945082/Put/seqid=0 2024-11-20T06:42:26,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742369_1545 (size=12001) 2024-11-20T06:42:26,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085006883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085006883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:26,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085006894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:27,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/4495ccab2ffe45caa464b91e867f3089 2024-11-20T06:42:27,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bffc3eabcbf947b08b5f791dfd2e9129 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bffc3eabcbf947b08b5f791dfd2e9129 2024-11-20T06:42:27,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bffc3eabcbf947b08b5f791dfd2e9129, entries=200, sequenceid=94, filesize=38.6 K 2024-11-20T06:42:27,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/97537c9c02f3417db4bf4f47c1650136 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/97537c9c02f3417db4bf4f47c1650136 2024-11-20T06:42:27,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/97537c9c02f3417db4bf4f47c1650136, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T06:42:27,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/4495ccab2ffe45caa464b91e867f3089 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/4495ccab2ffe45caa464b91e867f3089 2024-11-20T06:42:27,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/4495ccab2ffe45caa464b91e867f3089, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T06:42:27,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 03bb109fa60f64b2fccb76b2907dd474 in 851ms, sequenceid=94, compaction requested=true 2024-11-20T06:42:27,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:27,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:27,077 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:27,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:27,078 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:27,078 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:27,078 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/A is initiating minor compaction (all files) 2024-11-20T06:42:27,078 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/A in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:27,078 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/945f8965d015479ab454d5c8500b75ff, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/77f924a86f55465ab7f0023c8e566065, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bffc3eabcbf947b08b5f791dfd2e9129] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=99.2 K 2024-11-20T06:42:27,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:27,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:27,078 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:27,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:27,079 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/945f8965d015479ab454d5c8500b75ff, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/77f924a86f55465ab7f0023c8e566065, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bffc3eabcbf947b08b5f791dfd2e9129] 2024-11-20T06:42:27,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:27,079 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 945f8965d015479ab454d5c8500b75ff, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732084942918 2024-11-20T06:42:27,079 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:27,079 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/B is initiating minor compaction (all files) 2024-11-20T06:42:27,079 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/B in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:27,079 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/c1eea16ce2244daebf64ffe32cbe9736, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/27d6f1f0b6214f229e3f244f01498248, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/97537c9c02f3417db4bf4f47c1650136] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=35.3 K 2024-11-20T06:42:27,079 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77f924a86f55465ab7f0023c8e566065, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732084943999 2024-11-20T06:42:27,079 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting c1eea16ce2244daebf64ffe32cbe9736, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732084942918 2024-11-20T06:42:27,080 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting bffc3eabcbf947b08b5f791dfd2e9129, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732084945081 2024-11-20T06:42:27,080 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 27d6f1f0b6214f229e3f244f01498248, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732084943999 2024-11-20T06:42:27,080 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 97537c9c02f3417db4bf4f47c1650136, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732084945081 2024-11-20T06:42:27,086 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:27,087 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#B#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:27,087 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/94748e993ac148f299eb45468baa9a4e is 50, key is test_row_0/B:col10/1732084945082/Put/seqid=0 2024-11-20T06:42:27,088 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411202a4b893ec4ab495992d5beda6fa8f413_03bb109fa60f64b2fccb76b2907dd474 store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:27,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742370_1546 (size=12207) 2024-11-20T06:42:27,090 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411202a4b893ec4ab495992d5beda6fa8f413_03bb109fa60f64b2fccb76b2907dd474, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:27,090 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202a4b893ec4ab495992d5beda6fa8f413_03bb109fa60f64b2fccb76b2907dd474 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:27,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742371_1547 (size=4469) 2024-11-20T06:42:27,094 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#A#compaction#471 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:27,095 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bb3c4fd8fd6e457e9fc46856c3253a36 is 175, key is test_row_0/A:col10/1732084945082/Put/seqid=0 2024-11-20T06:42:27,098 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/94748e993ac148f299eb45468baa9a4e as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/94748e993ac148f299eb45468baa9a4e 2024-11-20T06:42:27,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742372_1548 (size=31161) 2024-11-20T06:42:27,102 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/B of 03bb109fa60f64b2fccb76b2907dd474 into 94748e993ac148f299eb45468baa9a4e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:27,102 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:27,102 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/B, priority=13, startTime=1732084947078; duration=0sec 2024-11-20T06:42:27,102 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:27,102 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:B 2024-11-20T06:42:27,102 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:27,103 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:27,103 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/C is initiating minor compaction (all files) 2024-11-20T06:42:27,103 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/C in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:27,103 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/bb166f21124644c1b61d5925f2d6f246, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b23878f2c3b94a0cbf36b78ff1d6a858, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/4495ccab2ffe45caa464b91e867f3089] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=35.3 K 2024-11-20T06:42:27,104 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting bb166f21124644c1b61d5925f2d6f246, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732084942918 2024-11-20T06:42:27,105 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b23878f2c3b94a0cbf36b78ff1d6a858, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732084943999 2024-11-20T06:42:27,105 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 4495ccab2ffe45caa464b91e867f3089, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732084945081 2024-11-20T06:42:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:27,108 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:27,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T06:42:27,113 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#C#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:27,113 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/3b96b8e9f0344badbccf5c3ed5cff5ba is 50, key is test_row_0/C:col10/1732084945082/Put/seqid=0 2024-11-20T06:42:27,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e8197295871f48baa8df2516d877277c_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084946258/Put/seqid=0 2024-11-20T06:42:27,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742373_1549 (size=12207) 2024-11-20T06:42:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742374_1550 (size=12154) 2024-11-20T06:42:27,137 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:27,140 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e8197295871f48baa8df2516d877277c_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e8197295871f48baa8df2516d877277c_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:27,142 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/a43e0c4d04e844428d01163759a2be67, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:27,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/a43e0c4d04e844428d01163759a2be67 is 175, key is test_row_0/A:col10/1732084946258/Put/seqid=0 2024-11-20T06:42:27,145 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/3b96b8e9f0344badbccf5c3ed5cff5ba as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3b96b8e9f0344badbccf5c3ed5cff5ba 2024-11-20T06:42:27,152 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/C of 03bb109fa60f64b2fccb76b2907dd474 into 3b96b8e9f0344badbccf5c3ed5cff5ba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:27,152 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:27,153 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/C, priority=13, startTime=1732084947079; duration=0sec 2024-11-20T06:42:27,154 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:27,154 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:C 2024-11-20T06:42:27,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742375_1551 (size=30955) 2024-11-20T06:42:27,159 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/a43e0c4d04e844428d01163759a2be67 2024-11-20T06:42:27,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/792196b159cb45ef96b3692e14bdd70b is 50, key is test_row_0/B:col10/1732084946258/Put/seqid=0 2024-11-20T06:42:27,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085007175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:27,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742376_1552 (size=12001) 2024-11-20T06:42:27,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085007284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:27,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085007393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:27,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085007395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:27,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085007404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:27,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:27,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085007493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:27,504 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bb3c4fd8fd6e457e9fc46856c3253a36 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bb3c4fd8fd6e457e9fc46856c3253a36 2024-11-20T06:42:27,508 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/A of 03bb109fa60f64b2fccb76b2907dd474 into bb3c4fd8fd6e457e9fc46856c3253a36(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:27,508 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:27,508 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/A, priority=13, startTime=1732084947077; duration=0sec 2024-11-20T06:42:27,508 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:27,508 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:A 2024-11-20T06:42:27,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/792196b159cb45ef96b3692e14bdd70b 2024-11-20T06:42:27,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/5f35f129464c4d27972a6679fa6893df is 50, key is test_row_0/C:col10/1732084946258/Put/seqid=0 2024-11-20T06:42:27,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742377_1553 (size=12001) 2024-11-20T06:42:27,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085007804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:28,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/5f35f129464c4d27972a6679fa6893df 2024-11-20T06:42:28,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/a43e0c4d04e844428d01163759a2be67 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/a43e0c4d04e844428d01163759a2be67 2024-11-20T06:42:28,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/a43e0c4d04e844428d01163759a2be67, entries=150, sequenceid=117, filesize=30.2 K 2024-11-20T06:42:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/792196b159cb45ef96b3692e14bdd70b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/792196b159cb45ef96b3692e14bdd70b 2024-11-20T06:42:28,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/792196b159cb45ef96b3692e14bdd70b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T06:42:28,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/5f35f129464c4d27972a6679fa6893df as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5f35f129464c4d27972a6679fa6893df 2024-11-20T06:42:28,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5f35f129464c4d27972a6679fa6893df, entries=150, sequenceid=117, filesize=11.7 K 2024-11-20T06:42:28,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 03bb109fa60f64b2fccb76b2907dd474 in 925ms, sequenceid=117, compaction requested=false 2024-11-20T06:42:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T06:42:28,054 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:28,055 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-20T06:42:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T06:42:28,056 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:28,057 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:28,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T06:42:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:28,206 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:28,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T06:42:28,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T06:42:28,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:28,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c7d113ff1ea64c5e85976436225948e1_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084948205/Put/seqid=0 2024-11-20T06:42:28,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742378_1554 (size=14744) 2024-11-20T06:42:28,216 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:28,219 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c7d113ff1ea64c5e85976436225948e1_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c7d113ff1ea64c5e85976436225948e1_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:28,219 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/4e7e0f766cd244ddbda1e08f381ad511, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:28,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/4e7e0f766cd244ddbda1e08f381ad511 is 175, key is test_row_0/A:col10/1732084948205/Put/seqid=0 2024-11-20T06:42:28,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742379_1555 (size=39699) 2024-11-20T06:42:28,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085008281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:28,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085008314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:28,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T06:42:28,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:28,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T06:42:28,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085008394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:28,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085008404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:28,407 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:28,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085008414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:28,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085008414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:28,417 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:28,417 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:28,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T06:42:28,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:28,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085008604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:28,624 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/4e7e0f766cd244ddbda1e08f381ad511 2024-11-20T06:42:28,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/2dffa7d558364d7194713a22b605c81a is 50, key is test_row_0/B:col10/1732084948205/Put/seqid=0 2024-11-20T06:42:28,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742380_1556 (size=12101) 2024-11-20T06:42:28,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/2dffa7d558364d7194713a22b605c81a 2024-11-20T06:42:28,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/b008fb7b7cef478993e15d77f8cc4eeb is 50, key is test_row_0/C:col10/1732084948205/Put/seqid=0 2024-11-20T06:42:28,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T06:42:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:28,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742381_1557 (size=12101) 2024-11-20T06:42:28,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/b008fb7b7cef478993e15d77f8cc4eeb 2024-11-20T06:42:28,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/4e7e0f766cd244ddbda1e08f381ad511 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/4e7e0f766cd244ddbda1e08f381ad511 2024-11-20T06:42:28,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/4e7e0f766cd244ddbda1e08f381ad511, entries=200, sequenceid=134, filesize=38.8 K 2024-11-20T06:42:28,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/2dffa7d558364d7194713a22b605c81a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2dffa7d558364d7194713a22b605c81a 2024-11-20T06:42:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T06:42:28,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2dffa7d558364d7194713a22b605c81a, entries=150, sequenceid=134, filesize=11.8 K 2024-11-20T06:42:28,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/b008fb7b7cef478993e15d77f8cc4eeb as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b008fb7b7cef478993e15d77f8cc4eeb 2024-11-20T06:42:28,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b008fb7b7cef478993e15d77f8cc4eeb, entries=150, sequenceid=134, filesize=11.8 K 2024-11-20T06:42:28,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 03bb109fa60f64b2fccb76b2907dd474 in 492ms, sequenceid=134, compaction requested=true 2024-11-20T06:42:28,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:28,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:28,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:28,698 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:28,698 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:28,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:28,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:28,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:28,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:28,699 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:28,699 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:28,699 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/A is initiating minor compaction (all files) 2024-11-20T06:42:28,699 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/B is initiating minor compaction (all files) 2024-11-20T06:42:28,699 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/B in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,699 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/A in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,699 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bb3c4fd8fd6e457e9fc46856c3253a36, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/a43e0c4d04e844428d01163759a2be67, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/4e7e0f766cd244ddbda1e08f381ad511] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=99.4 K 2024-11-20T06:42:28,699 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/94748e993ac148f299eb45468baa9a4e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/792196b159cb45ef96b3692e14bdd70b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2dffa7d558364d7194713a22b605c81a] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=35.5 K 2024-11-20T06:42:28,699 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,699 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 94748e993ac148f299eb45468baa9a4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732084945081 2024-11-20T06:42:28,700 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 792196b159cb45ef96b3692e14bdd70b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732084946257 2024-11-20T06:42:28,700 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dffa7d558364d7194713a22b605c81a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732084947146 2024-11-20T06:42:28,699 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bb3c4fd8fd6e457e9fc46856c3253a36, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/a43e0c4d04e844428d01163759a2be67, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/4e7e0f766cd244ddbda1e08f381ad511] 2024-11-20T06:42:28,705 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb3c4fd8fd6e457e9fc46856c3253a36, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732084945081 2024-11-20T06:42:28,705 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting a43e0c4d04e844428d01163759a2be67, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732084946257 2024-11-20T06:42:28,706 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e7e0f766cd244ddbda1e08f381ad511, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732084947146 2024-11-20T06:42:28,707 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:28,708 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/3598cc27bf81442faa63558bb8e5978f is 50, key is test_row_0/B:col10/1732084948205/Put/seqid=0 2024-11-20T06:42:28,726 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:28,731 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120a74f2efa6f30465c8a0783c6d55d92c9_03bb109fa60f64b2fccb76b2907dd474 store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:28,733 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120a74f2efa6f30465c8a0783c6d55d92c9_03bb109fa60f64b2fccb76b2907dd474, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:28,733 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a74f2efa6f30465c8a0783c6d55d92c9_03bb109fa60f64b2fccb76b2907dd474 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:28,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742382_1558 (size=12409) 2024-11-20T06:42:28,744 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/3598cc27bf81442faa63558bb8e5978f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/3598cc27bf81442faa63558bb8e5978f 2024-11-20T06:42:28,749 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/B of 03bb109fa60f64b2fccb76b2907dd474 into 3598cc27bf81442faa63558bb8e5978f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:28,749 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:28,749 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/B, priority=13, startTime=1732084948698; duration=0sec 2024-11-20T06:42:28,749 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:28,749 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:B 2024-11-20T06:42:28,749 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:28,750 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:28,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742383_1559 (size=4469) 2024-11-20T06:42:28,752 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/C is initiating minor compaction (all files) 2024-11-20T06:42:28,752 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/C in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,752 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3b96b8e9f0344badbccf5c3ed5cff5ba, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5f35f129464c4d27972a6679fa6893df, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b008fb7b7cef478993e15d77f8cc4eeb] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=35.5 K 2024-11-20T06:42:28,753 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#A#compaction#481 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:28,753 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b96b8e9f0344badbccf5c3ed5cff5ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732084945081 2024-11-20T06:42:28,753 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/7476366812e34b98bcc26f1a3cf36c5d is 175, key is test_row_0/A:col10/1732084948205/Put/seqid=0 2024-11-20T06:42:28,753 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f35f129464c4d27972a6679fa6893df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732084946257 2024-11-20T06:42:28,754 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b008fb7b7cef478993e15d77f8cc4eeb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732084947146 2024-11-20T06:42:28,765 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:28,765 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/47394d73fcdd4d80a1ae1fefbc609658 is 50, key is test_row_0/C:col10/1732084948205/Put/seqid=0 2024-11-20T06:42:28,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742384_1560 (size=31363) 2024-11-20T06:42:28,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742385_1561 (size=12409) 2024-11-20T06:42:28,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T06:42:28,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:28,820 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T06:42:28,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205c34a911a840403ab453cb782cfe6d61_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084948266/Put/seqid=0 2024-11-20T06:42:28,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742386_1562 (size=12304) 2024-11-20T06:42:28,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:28,864 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205c34a911a840403ab453cb782cfe6d61_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205c34a911a840403ab453cb782cfe6d61_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:28,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/024ee345dc454d268636a50d0e02ccd6, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:28,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/024ee345dc454d268636a50d0e02ccd6 is 175, key is test_row_0/A:col10/1732084948266/Put/seqid=0 2024-11-20T06:42:28,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742387_1563 (size=31105) 2024-11-20T06:42:28,887 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/024ee345dc454d268636a50d0e02ccd6 2024-11-20T06:42:28,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/d9c071e487834bcf8b6d9ddcc63b4b18 is 50, key is test_row_0/B:col10/1732084948266/Put/seqid=0 2024-11-20T06:42:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:28,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:28,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742388_1564 (size=12151) 2024-11-20T06:42:28,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085008961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:29,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085009073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:29,172 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/7476366812e34b98bcc26f1a3cf36c5d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/7476366812e34b98bcc26f1a3cf36c5d 2024-11-20T06:42:29,176 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/A of 03bb109fa60f64b2fccb76b2907dd474 into 7476366812e34b98bcc26f1a3cf36c5d(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:29,176 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:29,176 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/A, priority=13, startTime=1732084948698; duration=0sec 2024-11-20T06:42:29,176 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:29,176 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:A 2024-11-20T06:42:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T06:42:29,219 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/47394d73fcdd4d80a1ae1fefbc609658 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/47394d73fcdd4d80a1ae1fefbc609658 2024-11-20T06:42:29,223 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/C of 03bb109fa60f64b2fccb76b2907dd474 into 47394d73fcdd4d80a1ae1fefbc609658(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:29,223 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:29,223 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/C, priority=13, startTime=1732084948698; duration=0sec 2024-11-20T06:42:29,223 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:29,223 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:C 2024-11-20T06:42:29,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085009284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:29,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:29,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085009323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:29,325 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:29,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:29,328 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/d9c071e487834bcf8b6d9ddcc63b4b18 2024-11-20T06:42:29,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/5575a7a3599f4151ad7428eb334b8b11 is 50, key is test_row_0/C:col10/1732084948266/Put/seqid=0 2024-11-20T06:42:29,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742389_1565 (size=12151) 2024-11-20T06:42:29,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085009593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:29,740 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/5575a7a3599f4151ad7428eb334b8b11 2024-11-20T06:42:29,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/024ee345dc454d268636a50d0e02ccd6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/024ee345dc454d268636a50d0e02ccd6 2024-11-20T06:42:29,745 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/024ee345dc454d268636a50d0e02ccd6, entries=150, sequenceid=156, filesize=30.4 K 2024-11-20T06:42:29,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/d9c071e487834bcf8b6d9ddcc63b4b18 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d9c071e487834bcf8b6d9ddcc63b4b18 2024-11-20T06:42:29,748 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d9c071e487834bcf8b6d9ddcc63b4b18, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T06:42:29,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/5575a7a3599f4151ad7428eb334b8b11 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5575a7a3599f4151ad7428eb334b8b11 2024-11-20T06:42:29,751 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5575a7a3599f4151ad7428eb334b8b11, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T06:42:29,752 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 03bb109fa60f64b2fccb76b2907dd474 in 932ms, sequenceid=156, compaction requested=false 2024-11-20T06:42:29,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:29,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:29,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-20T06:42:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-20T06:42:29,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-20T06:42:29,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6960 sec 2024-11-20T06:42:29,756 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.7000 sec 2024-11-20T06:42:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:30,107 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:30,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T06:42:30,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206c73c4d8a9f241f39c1d32d7475dfc6b_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084948956/Put/seqid=0 2024-11-20T06:42:30,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742390_1566 (size=14794) 2024-11-20T06:42:30,116 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:30,123 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206c73c4d8a9f241f39c1d32d7475dfc6b_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c73c4d8a9f241f39c1d32d7475dfc6b_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:30,124 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/d87c39436fcb4d2aaa7d2b19d0356444, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:30,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/d87c39436fcb4d2aaa7d2b19d0356444 is 175, key is test_row_0/A:col10/1732084948956/Put/seqid=0 2024-11-20T06:42:30,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742391_1567 (size=39749) 2024-11-20T06:42:30,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085010189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T06:42:30,204 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:30,205 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-20T06:42:30,207 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T06:42:30,207 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:30,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:30,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085010304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T06:42:30,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T06:42:30,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:30,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:30,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:30,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085010414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:30,416 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:30,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085010433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:30,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085010433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:30,435 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:30,436 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:30,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T06:42:30,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:30,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:30,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:30,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085010514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T06:42:30,528 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/d87c39436fcb4d2aaa7d2b19d0356444 2024-11-20T06:42:30,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/7e7c4339ade74e81aea53a1e94c79a0f is 50, key is test_row_0/B:col10/1732084948956/Put/seqid=0 2024-11-20T06:42:30,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742392_1568 (size=12151) 2024-11-20T06:42:30,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T06:42:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:30,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T06:42:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:30,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085010824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:30,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T06:42:30,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/7e7c4339ade74e81aea53a1e94c79a0f 2024-11-20T06:42:30,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/f84d60e9f3914cff9d4c71e23281dc85 is 50, key is test_row_0/C:col10/1732084948956/Put/seqid=0 2024-11-20T06:42:30,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742393_1569 (size=12151) 2024-11-20T06:42:30,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T06:42:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:30,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:31,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T06:42:31,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:31,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:31,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:31,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:31,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:31,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T06:42:31,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:31,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:31,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:31,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:31,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:31,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085011334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T06:42:31,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/f84d60e9f3914cff9d4c71e23281dc85 2024-11-20T06:42:31,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/d87c39436fcb4d2aaa7d2b19d0356444 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/d87c39436fcb4d2aaa7d2b19d0356444 2024-11-20T06:42:31,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/d87c39436fcb4d2aaa7d2b19d0356444, entries=200, sequenceid=174, filesize=38.8 K 2024-11-20T06:42:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/7e7c4339ade74e81aea53a1e94c79a0f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/7e7c4339ade74e81aea53a1e94c79a0f 2024-11-20T06:42:31,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/7e7c4339ade74e81aea53a1e94c79a0f, entries=150, sequenceid=174, filesize=11.9 K 2024-11-20T06:42:31,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/f84d60e9f3914cff9d4c71e23281dc85 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/f84d60e9f3914cff9d4c71e23281dc85 2024-11-20T06:42:31,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085011353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:31,358 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:31,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/f84d60e9f3914cff9d4c71e23281dc85, entries=150, sequenceid=174, filesize=11.9 K 2024-11-20T06:42:31,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 03bb109fa60f64b2fccb76b2907dd474 in 1253ms, sequenceid=174, compaction requested=true 2024-11-20T06:42:31,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:31,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:31,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:31,361 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:31,361 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:31,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:31,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:31,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:31,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:31,362 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:31,362 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:31,362 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/B is initiating minor compaction (all files) 2024-11-20T06:42:31,362 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/A is initiating minor compaction (all files) 2024-11-20T06:42:31,362 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/A in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:31,362 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/B in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:31,362 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/3598cc27bf81442faa63558bb8e5978f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d9c071e487834bcf8b6d9ddcc63b4b18, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/7e7c4339ade74e81aea53a1e94c79a0f] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=35.9 K 2024-11-20T06:42:31,362 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/7476366812e34b98bcc26f1a3cf36c5d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/024ee345dc454d268636a50d0e02ccd6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/d87c39436fcb4d2aaa7d2b19d0356444] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=99.8 K 2024-11-20T06:42:31,362 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:31,362 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/7476366812e34b98bcc26f1a3cf36c5d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/024ee345dc454d268636a50d0e02ccd6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/d87c39436fcb4d2aaa7d2b19d0356444] 2024-11-20T06:42:31,362 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 3598cc27bf81442faa63558bb8e5978f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732084947146 2024-11-20T06:42:31,362 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7476366812e34b98bcc26f1a3cf36c5d, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732084947146 2024-11-20T06:42:31,363 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d9c071e487834bcf8b6d9ddcc63b4b18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732084948260 2024-11-20T06:42:31,363 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e7c4339ade74e81aea53a1e94c79a0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732084948941 2024-11-20T06:42:31,363 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024ee345dc454d268636a50d0e02ccd6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732084948260 2024-11-20T06:42:31,363 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting d87c39436fcb4d2aaa7d2b19d0356444, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732084948941 2024-11-20T06:42:31,372 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:31,373 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/29f8243e09a94d7ca791c7d6be88323a is 50, key is test_row_0/B:col10/1732084948956/Put/seqid=0 2024-11-20T06:42:31,375 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:31,387 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120a6e6c2c34f964b8e88c5abc3e729f889_03bb109fa60f64b2fccb76b2907dd474 store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:31,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742394_1570 (size=12561) 2024-11-20T06:42:31,389 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120a6e6c2c34f964b8e88c5abc3e729f889_03bb109fa60f64b2fccb76b2907dd474, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:31,389 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a6e6c2c34f964b8e88c5abc3e729f889_03bb109fa60f64b2fccb76b2907dd474 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:31,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742395_1571 (size=4469) 2024-11-20T06:42:31,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T06:42:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:31,427 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:42:31,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b2d5a6d51b004728b08a99701f820e9a_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084950143/Put/seqid=0 2024-11-20T06:42:31,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742396_1572 (size=12304) 2024-11-20T06:42:31,791 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/29f8243e09a94d7ca791c7d6be88323a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/29f8243e09a94d7ca791c7d6be88323a 2024-11-20T06:42:31,795 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/B of 03bb109fa60f64b2fccb76b2907dd474 into 29f8243e09a94d7ca791c7d6be88323a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:31,795 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:31,795 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/B, priority=13, startTime=1732084951361; duration=0sec 2024-11-20T06:42:31,795 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:31,795 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:B 2024-11-20T06:42:31,795 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:31,795 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#A#compaction#490 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:31,796 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/252ea1249df540ff9005a6c6cce20710 is 175, key is test_row_0/A:col10/1732084948956/Put/seqid=0 2024-11-20T06:42:31,796 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:31,796 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/C is initiating minor compaction (all files) 2024-11-20T06:42:31,796 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/C in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:31,796 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/47394d73fcdd4d80a1ae1fefbc609658, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5575a7a3599f4151ad7428eb334b8b11, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/f84d60e9f3914cff9d4c71e23281dc85] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=35.9 K 2024-11-20T06:42:31,796 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 47394d73fcdd4d80a1ae1fefbc609658, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732084947146 2024-11-20T06:42:31,797 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5575a7a3599f4151ad7428eb334b8b11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732084948260 2024-11-20T06:42:31,797 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting f84d60e9f3914cff9d4c71e23281dc85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732084948941 2024-11-20T06:42:31,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742397_1573 (size=31515) 2024-11-20T06:42:31,802 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:31,802 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/d4f5d325201d44f7ab1af24372524f71 is 50, key is test_row_0/C:col10/1732084948956/Put/seqid=0 2024-11-20T06:42:31,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742398_1574 (size=12561) 2024-11-20T06:42:31,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:31,837 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b2d5a6d51b004728b08a99701f820e9a_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b2d5a6d51b004728b08a99701f820e9a_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:31,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/846ca60c6bc2483fa77a885b348d03d3, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:31,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/846ca60c6bc2483fa77a885b348d03d3 is 175, key is test_row_0/A:col10/1732084950143/Put/seqid=0 2024-11-20T06:42:31,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742399_1575 (size=31105) 2024-11-20T06:42:32,202 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/252ea1249df540ff9005a6c6cce20710 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/252ea1249df540ff9005a6c6cce20710 2024-11-20T06:42:32,205 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/A of 03bb109fa60f64b2fccb76b2907dd474 into 252ea1249df540ff9005a6c6cce20710(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:32,205 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:32,205 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/A, priority=13, startTime=1732084951360; duration=0sec 2024-11-20T06:42:32,205 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:32,205 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:A 2024-11-20T06:42:32,208 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/d4f5d325201d44f7ab1af24372524f71 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/d4f5d325201d44f7ab1af24372524f71 2024-11-20T06:42:32,212 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/C of 03bb109fa60f64b2fccb76b2907dd474 into d4f5d325201d44f7ab1af24372524f71(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:32,212 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:32,212 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/C, priority=13, startTime=1732084951361; duration=0sec 2024-11-20T06:42:32,212 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:32,212 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:C 2024-11-20T06:42:32,242 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/846ca60c6bc2483fa77a885b348d03d3 2024-11-20T06:42:32,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/f2a353945b754f9ba3022d1604b9e7f4 is 50, key is test_row_0/B:col10/1732084950143/Put/seqid=0 2024-11-20T06:42:32,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742400_1576 (size=12151) 2024-11-20T06:42:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:32,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T06:42:32,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085012384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:32,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085012494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:32,651 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/f2a353945b754f9ba3022d1604b9e7f4 2024-11-20T06:42:32,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a2f579e6fa30453d846d714e50373e49 is 50, key is test_row_0/C:col10/1732084950143/Put/seqid=0 2024-11-20T06:42:32,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742401_1577 (size=12151) 2024-11-20T06:42:32,660 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a2f579e6fa30453d846d714e50373e49 2024-11-20T06:42:32,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/846ca60c6bc2483fa77a885b348d03d3 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/846ca60c6bc2483fa77a885b348d03d3 2024-11-20T06:42:32,665 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/846ca60c6bc2483fa77a885b348d03d3, entries=150, sequenceid=194, filesize=30.4 K 2024-11-20T06:42:32,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/f2a353945b754f9ba3022d1604b9e7f4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/f2a353945b754f9ba3022d1604b9e7f4 2024-11-20T06:42:32,668 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/f2a353945b754f9ba3022d1604b9e7f4, entries=150, sequenceid=194, filesize=11.9 K 2024-11-20T06:42:32,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a2f579e6fa30453d846d714e50373e49 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f579e6fa30453d846d714e50373e49 2024-11-20T06:42:32,671 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f579e6fa30453d846d714e50373e49, entries=150, sequenceid=194, filesize=11.9 K 2024-11-20T06:42:32,672 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 03bb109fa60f64b2fccb76b2907dd474 in 1245ms, sequenceid=194, compaction requested=false 2024-11-20T06:42:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-20T06:42:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-20T06:42:32,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-20T06:42:32,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4660 sec 2024-11-20T06:42:32,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.4710 sec 2024-11-20T06:42:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:32,706 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:32,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-20T06:42:32,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120deff2ef9e46d47e0b8e03a3298977a8b_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084952379/Put/seqid=0 2024-11-20T06:42:32,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742402_1578 (size=12304) 2024-11-20T06:42:32,719 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:32,721 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120deff2ef9e46d47e0b8e03a3298977a8b_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120deff2ef9e46d47e0b8e03a3298977a8b_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:32,722 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bc974041ca5d457dbfe0684f3080dcdc, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:32,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bc974041ca5d457dbfe0684f3080dcdc is 175, key is test_row_0/A:col10/1732084952379/Put/seqid=0 2024-11-20T06:42:32,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742403_1579 (size=31105) 2024-11-20T06:42:32,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085012770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:32,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:32,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:32,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:32,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085012883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:32,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:32,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:32,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:33,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085013094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:33,126 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bc974041ca5d457dbfe0684f3080dcdc 2024-11-20T06:42:33,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/ed5e1e42e8154f48a63253d95b9d83e2 is 50, key is test_row_0/B:col10/1732084952379/Put/seqid=0 2024-11-20T06:42:33,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742404_1580 (size=12151) 2024-11-20T06:42:33,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085013404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:33,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:33,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/ed5e1e42e8154f48a63253d95b9d83e2 2024-11-20T06:42:33,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/3c55acd6b24d4cd383a68d073a5ca4ab is 50, key is test_row_0/C:col10/1732084952379/Put/seqid=0 2024-11-20T06:42:33,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742405_1581 (size=12151) 2024-11-20T06:42:33,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/3c55acd6b24d4cd383a68d073a5ca4ab 2024-11-20T06:42:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bc974041ca5d457dbfe0684f3080dcdc as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bc974041ca5d457dbfe0684f3080dcdc 2024-11-20T06:42:33,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bc974041ca5d457dbfe0684f3080dcdc, entries=150, sequenceid=214, filesize=30.4 K 2024-11-20T06:42:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/ed5e1e42e8154f48a63253d95b9d83e2 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/ed5e1e42e8154f48a63253d95b9d83e2 2024-11-20T06:42:33,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/ed5e1e42e8154f48a63253d95b9d83e2, entries=150, sequenceid=214, filesize=11.9 K 2024-11-20T06:42:33,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/3c55acd6b24d4cd383a68d073a5ca4ab as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3c55acd6b24d4cd383a68d073a5ca4ab 2024-11-20T06:42:33,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3c55acd6b24d4cd383a68d073a5ca4ab, entries=150, sequenceid=214, filesize=11.9 K 2024-11-20T06:42:33,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 03bb109fa60f64b2fccb76b2907dd474 in 849ms, sequenceid=214, compaction requested=true 2024-11-20T06:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:33,556 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:33,556 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:33,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:33,556 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:33,556 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:33,557 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/A is initiating minor compaction (all files) 2024-11-20T06:42:33,557 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/B is initiating minor compaction (all files) 2024-11-20T06:42:33,557 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/B in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:33,557 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/A in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:33,557 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/252ea1249df540ff9005a6c6cce20710, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/846ca60c6bc2483fa77a885b348d03d3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bc974041ca5d457dbfe0684f3080dcdc] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=91.5 K 2024-11-20T06:42:33,557 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/29f8243e09a94d7ca791c7d6be88323a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/f2a353945b754f9ba3022d1604b9e7f4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/ed5e1e42e8154f48a63253d95b9d83e2] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=36.0 K 2024-11-20T06:42:33,557 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:33,557 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/252ea1249df540ff9005a6c6cce20710, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/846ca60c6bc2483fa77a885b348d03d3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bc974041ca5d457dbfe0684f3080dcdc] 2024-11-20T06:42:33,557 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 29f8243e09a94d7ca791c7d6be88323a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732084948941 2024-11-20T06:42:33,557 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 252ea1249df540ff9005a6c6cce20710, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732084948941 2024-11-20T06:42:33,557 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a353945b754f9ba3022d1604b9e7f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732084950143 2024-11-20T06:42:33,557 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 846ca60c6bc2483fa77a885b348d03d3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732084950143 2024-11-20T06:42:33,557 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting ed5e1e42e8154f48a63253d95b9d83e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732084952379 2024-11-20T06:42:33,558 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc974041ca5d457dbfe0684f3080dcdc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732084952379 2024-11-20T06:42:33,562 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:33,564 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#B#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:33,565 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/2343501f78044f56919998df60495a7a is 50, key is test_row_0/B:col10/1732084952379/Put/seqid=0 2024-11-20T06:42:33,565 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120defe47fad640417f89effef7aacb3c72_03bb109fa60f64b2fccb76b2907dd474 store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:33,567 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120defe47fad640417f89effef7aacb3c72_03bb109fa60f64b2fccb76b2907dd474, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:33,567 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120defe47fad640417f89effef7aacb3c72_03bb109fa60f64b2fccb76b2907dd474 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:33,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742406_1582 (size=12663) 2024-11-20T06:42:33,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742407_1583 (size=4469) 2024-11-20T06:42:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:33,915 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:33,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-20T06:42:33,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e4f31358749746f8803a195bc7447105_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084952769/Put/seqid=0 2024-11-20T06:42:33,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742408_1584 (size=14794) 2024-11-20T06:42:33,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085013970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:33,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:33,979 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/2343501f78044f56919998df60495a7a as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2343501f78044f56919998df60495a7a 2024-11-20T06:42:33,983 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/B of 03bb109fa60f64b2fccb76b2907dd474 into 2343501f78044f56919998df60495a7a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:33,983 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:33,983 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/B, priority=13, startTime=1732084953556; duration=0sec 2024-11-20T06:42:33,983 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:33,983 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:B 2024-11-20T06:42:33,983 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:33,984 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:33,984 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/C is initiating minor compaction (all files) 2024-11-20T06:42:33,984 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/C in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:33,984 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/d4f5d325201d44f7ab1af24372524f71, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f579e6fa30453d846d714e50373e49, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3c55acd6b24d4cd383a68d073a5ca4ab] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=36.0 K 2024-11-20T06:42:33,984 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting d4f5d325201d44f7ab1af24372524f71, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732084948941 2024-11-20T06:42:33,984 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f579e6fa30453d846d714e50373e49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732084950143 2024-11-20T06:42:33,985 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c55acd6b24d4cd383a68d073a5ca4ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732084952379 2024-11-20T06:42:33,990 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#C#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:33,990 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#A#compaction#498 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:33,990 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/de100b0dae80457082c25fc67f5775fd is 175, key is test_row_0/A:col10/1732084952379/Put/seqid=0 2024-11-20T06:42:33,990 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/37b0c1709a1f4e5d8c8f927782999d12 is 50, key is test_row_0/C:col10/1732084952379/Put/seqid=0 2024-11-20T06:42:33,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742409_1585 (size=12663) 2024-11-20T06:42:33,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742410_1586 (size=31617) 2024-11-20T06:42:34,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085014083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:34,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085014294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:34,324 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:34,327 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e4f31358749746f8803a195bc7447105_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e4f31358749746f8803a195bc7447105_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:34,330 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/b40aeeec431847afb98d4502acc35171, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:34,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/b40aeeec431847afb98d4502acc35171 is 175, key is test_row_0/A:col10/1732084952769/Put/seqid=0 2024-11-20T06:42:34,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742411_1587 (size=39749) 2024-11-20T06:42:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T06:42:34,374 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:34,374 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-20T06:42:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T06:42:34,376 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:34,376 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:34,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:34,397 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/37b0c1709a1f4e5d8c8f927782999d12 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/37b0c1709a1f4e5d8c8f927782999d12 2024-11-20T06:42:34,397 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/de100b0dae80457082c25fc67f5775fd as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/de100b0dae80457082c25fc67f5775fd 2024-11-20T06:42:34,401 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/A of 03bb109fa60f64b2fccb76b2907dd474 into de100b0dae80457082c25fc67f5775fd(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:34,401 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:34,401 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/A, priority=13, startTime=1732084953556; duration=0sec 2024-11-20T06:42:34,401 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:34,401 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:A 2024-11-20T06:42:34,401 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/C of 03bb109fa60f64b2fccb76b2907dd474 into 37b0c1709a1f4e5d8c8f927782999d12(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:34,401 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:34,401 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/C, priority=13, startTime=1732084953556; duration=0sec 2024-11-20T06:42:34,401 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:34,401 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:C 2024-11-20T06:42:34,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43452 deadline: 1732085014454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:34,456 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_1' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:34,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43384 deadline: 1732085014463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:34,466 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:34,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T06:42:34,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43446 deadline: 1732085014484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:34,486 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_0' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:34,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:34,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T06:42:34,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:34,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:34,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:34,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085014604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:34,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T06:42:34,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:34,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:34,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:34,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T06:42:34,734 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/b40aeeec431847afb98d4502acc35171 2024-11-20T06:42:34,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/0f7f5f9260364583b11d377cff0b24ae is 50, key is test_row_0/B:col10/1732084952769/Put/seqid=0 2024-11-20T06:42:34,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742412_1588 (size=12151) 2024-11-20T06:42:34,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/0f7f5f9260364583b11d377cff0b24ae 2024-11-20T06:42:34,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/81d49a1a7ed148e595e55af97425009b is 50, key is test_row_0/C:col10/1732084952769/Put/seqid=0 2024-11-20T06:42:34,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742413_1589 (size=12151) 2024-11-20T06:42:34,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T06:42:34,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:34,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:34,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:34,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T06:42:34,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:34,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T06:42:35,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085015114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:35,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:35,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:35,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:35,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T06:42:35,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:35,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:35,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:35,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:35,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:35,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/81d49a1a7ed148e595e55af97425009b 2024-11-20T06:42:35,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/b40aeeec431847afb98d4502acc35171 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/b40aeeec431847afb98d4502acc35171 2024-11-20T06:42:35,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/b40aeeec431847afb98d4502acc35171, entries=200, sequenceid=234, filesize=38.8 K 2024-11-20T06:42:35,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/0f7f5f9260364583b11d377cff0b24ae as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/0f7f5f9260364583b11d377cff0b24ae 2024-11-20T06:42:35,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/0f7f5f9260364583b11d377cff0b24ae, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T06:42:35,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/81d49a1a7ed148e595e55af97425009b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/81d49a1a7ed148e595e55af97425009b 2024-11-20T06:42:35,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/81d49a1a7ed148e595e55af97425009b, entries=150, sequenceid=234, filesize=11.9 K 2024-11-20T06:42:35,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 03bb109fa60f64b2fccb76b2907dd474 in 1248ms, sequenceid=234, compaction requested=false 2024-11-20T06:42:35,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:35,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T06:42:35,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:35,292 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-20T06:42:35,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207d9d3c868fe442ff9efa481e580c5c9d_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084953964/Put/seqid=0 2024-11-20T06:42:35,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742414_1590 (size=12304) 2024-11-20T06:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:35,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085015457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T06:42:35,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085015564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:35,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:35,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:35,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:35,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:35,703 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207d9d3c868fe442ff9efa481e580c5c9d_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207d9d3c868fe442ff9efa481e580c5c9d_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:35,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/fbee1ae95ab3444abc5422dea4b6db8d, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/fbee1ae95ab3444abc5422dea4b6db8d is 175, key is test_row_0/A:col10/1732084953964/Put/seqid=0 2024-11-20T06:42:35,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742415_1591 (size=31105) 2024-11-20T06:42:35,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085015774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:35,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:35,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:35,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:36,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085016084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:36,108 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/fbee1ae95ab3444abc5422dea4b6db8d 2024-11-20T06:42:36,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/73c5a2d0095e4ea9b4ac2b427201f78f is 50, key is test_row_0/B:col10/1732084953964/Put/seqid=0 2024-11-20T06:42:36,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742416_1592 (size=12151) 2024-11-20T06:42:36,129 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/73c5a2d0095e4ea9b4ac2b427201f78f 2024-11-20T06:42:36,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/aa59d273508e45908ba2dcc60d0b7e42 is 50, key is test_row_0/C:col10/1732084953964/Put/seqid=0 2024-11-20T06:42:36,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085016134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:36,137 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:36,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742417_1593 (size=12151) 2024-11-20T06:42:36,145 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/aa59d273508e45908ba2dcc60d0b7e42 2024-11-20T06:42:36,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/fbee1ae95ab3444abc5422dea4b6db8d as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/fbee1ae95ab3444abc5422dea4b6db8d 2024-11-20T06:42:36,153 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/fbee1ae95ab3444abc5422dea4b6db8d, entries=150, sequenceid=253, filesize=30.4 K 2024-11-20T06:42:36,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/73c5a2d0095e4ea9b4ac2b427201f78f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/73c5a2d0095e4ea9b4ac2b427201f78f 2024-11-20T06:42:36,158 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/73c5a2d0095e4ea9b4ac2b427201f78f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-20T06:42:36,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/aa59d273508e45908ba2dcc60d0b7e42 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/aa59d273508e45908ba2dcc60d0b7e42 2024-11-20T06:42:36,162 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/aa59d273508e45908ba2dcc60d0b7e42, entries=150, sequenceid=253, filesize=11.9 K 2024-11-20T06:42:36,163 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 03bb109fa60f64b2fccb76b2907dd474 in 872ms, sequenceid=253, compaction requested=true 2024-11-20T06:42:36,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:36,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:36,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-20T06:42:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-20T06:42:36,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-20T06:42:36,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7880 sec 2024-11-20T06:42:36,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.7920 sec 2024-11-20T06:42:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T06:42:36,524 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:36,524 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-20T06:42:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T06:42:36,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:36,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:36,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:36,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:36,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-20T06:42:36,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209c7d69d4410648249aa3633d7ee24636_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084955438/Put/seqid=0 2024-11-20T06:42:36,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742418_1594 (size=12454) 2024-11-20T06:42:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T06:42:36,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085016650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:36,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:36,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:36,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:36,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:36,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:36,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:36,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085016763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:36,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:36,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:36,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:36,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:36,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:36,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T06:42:36,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:36,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085016974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:36,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:36,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:36,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:37,025 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209c7d69d4410648249aa3633d7ee24636_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209c7d69d4410648249aa3633d7ee24636_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:37,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/dda7326ac94a41498dd16c708262194b, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:37,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/dda7326ac94a41498dd16c708262194b is 175, key is test_row_0/A:col10/1732084955438/Put/seqid=0 2024-11-20T06:42:37,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742419_1595 (size=31255) 2024-11-20T06:42:37,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T06:42:37,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085017283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:37,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:37,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:37,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,430 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/dda7326ac94a41498dd16c708262194b 2024-11-20T06:42:37,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/a9a652e0e17244b6af9ad7d21ae1c6df is 50, key is test_row_0/B:col10/1732084955438/Put/seqid=0 2024-11-20T06:42:37,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742420_1596 (size=12301) 2024-11-20T06:42:37,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:37,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:37,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:37,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:37,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T06:42:37,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:37,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:37,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085017794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:37,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:37,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:37,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:37,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/a9a652e0e17244b6af9ad7d21ae1c6df 2024-11-20T06:42:37,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/1cc33171163447c58b82b7202042c953 is 50, key is test_row_0/C:col10/1732084955438/Put/seqid=0 2024-11-20T06:42:37,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742421_1597 (size=12301) 2024-11-20T06:42:37,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:37,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:38,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:38,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:38,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:38,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:38,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:38,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:38,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1732085018144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:38,148 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 61c2066a8a0f,33847,1732084794617 for 'test_row_2' in 03bb109fa60f64b2fccb76b2907dd474 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-20T06:42:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:38,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:38,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:38,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:38,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:38,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:38,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:38,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:38,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/1cc33171163447c58b82b7202042c953 2024-11-20T06:42:38,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/dda7326ac94a41498dd16c708262194b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/dda7326ac94a41498dd16c708262194b 2024-11-20T06:42:38,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/dda7326ac94a41498dd16c708262194b, entries=150, sequenceid=274, filesize=30.5 K 2024-11-20T06:42:38,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/a9a652e0e17244b6af9ad7d21ae1c6df as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/a9a652e0e17244b6af9ad7d21ae1c6df 2024-11-20T06:42:38,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/a9a652e0e17244b6af9ad7d21ae1c6df, entries=150, sequenceid=274, filesize=12.0 K 2024-11-20T06:42:38,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/1cc33171163447c58b82b7202042c953 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/1cc33171163447c58b82b7202042c953 2024-11-20T06:42:38,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/1cc33171163447c58b82b7202042c953, entries=150, sequenceid=274, filesize=12.0 K 2024-11-20T06:42:38,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 03bb109fa60f64b2fccb76b2907dd474 in 1662ms, sequenceid=274, compaction requested=true 2024-11-20T06:42:38,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:38,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:38,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:38,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:38,260 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:42:38,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:38,260 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:42:38,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:38,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:38,261 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:42:38,261 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:42:38,261 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/B is initiating minor compaction (all files) 2024-11-20T06:42:38,261 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/A is initiating minor compaction (all files) 2024-11-20T06:42:38,261 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/B in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:38,261 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/A in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:38,261 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2343501f78044f56919998df60495a7a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/0f7f5f9260364583b11d377cff0b24ae, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/73c5a2d0095e4ea9b4ac2b427201f78f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/a9a652e0e17244b6af9ad7d21ae1c6df] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=48.1 K 2024-11-20T06:42:38,261 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/de100b0dae80457082c25fc67f5775fd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/b40aeeec431847afb98d4502acc35171, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/fbee1ae95ab3444abc5422dea4b6db8d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/dda7326ac94a41498dd16c708262194b] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=130.6 K 2024-11-20T06:42:38,261 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:38,261 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/de100b0dae80457082c25fc67f5775fd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/b40aeeec431847afb98d4502acc35171, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/fbee1ae95ab3444abc5422dea4b6db8d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/dda7326ac94a41498dd16c708262194b] 2024-11-20T06:42:38,261 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 2343501f78044f56919998df60495a7a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732084952379 2024-11-20T06:42:38,261 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting de100b0dae80457082c25fc67f5775fd, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732084952379 2024-11-20T06:42:38,261 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7f5f9260364583b11d377cff0b24ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732084952761 2024-11-20T06:42:38,261 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40aeeec431847afb98d4502acc35171, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732084952745 2024-11-20T06:42:38,261 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 73c5a2d0095e4ea9b4ac2b427201f78f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732084953955 2024-11-20T06:42:38,262 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbee1ae95ab3444abc5422dea4b6db8d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732084953955 2024-11-20T06:42:38,262 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a652e0e17244b6af9ad7d21ae1c6df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732084955438 2024-11-20T06:42:38,262 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting dda7326ac94a41498dd16c708262194b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732084955438 2024-11-20T06:42:38,268 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#B#compaction#510 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:38,268 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/9729dbca00b34364a345ab5fb4bbf9ad is 50, key is test_row_0/B:col10/1732084955438/Put/seqid=0 2024-11-20T06:42:38,269 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:38,276 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120efdfcdc9985c4485ba5327d9e9c51f44_03bb109fa60f64b2fccb76b2907dd474 store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:38,277 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120efdfcdc9985c4485ba5327d9e9c51f44_03bb109fa60f64b2fccb76b2907dd474, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:38,277 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120efdfcdc9985c4485ba5327d9e9c51f44_03bb109fa60f64b2fccb76b2907dd474 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:38,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742422_1598 (size=12949) 2024-11-20T06:42:38,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742423_1599 (size=4469) 2024-11-20T06:42:38,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T06:42:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:38,356 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-20T06:42:38,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dde6168b72b44339a899d4c3046c581b_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084956648/Put/seqid=0 2024-11-20T06:42:38,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742424_1600 (size=12454) 2024-11-20T06:42:38,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:38,368 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dde6168b72b44339a899d4c3046c581b_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dde6168b72b44339a899d4c3046c581b_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:38,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/0c0e43e00710460d9a4cee31e9424e24, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:38,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/0c0e43e00710460d9a4cee31e9424e24 is 175, key is test_row_0/A:col10/1732084956648/Put/seqid=0 2024-11-20T06:42:38,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742425_1601 (size=31255) 2024-11-20T06:42:38,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T06:42:38,680 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/9729dbca00b34364a345ab5fb4bbf9ad as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9729dbca00b34364a345ab5fb4bbf9ad 2024-11-20T06:42:38,684 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/B of 03bb109fa60f64b2fccb76b2907dd474 into 9729dbca00b34364a345ab5fb4bbf9ad(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:38,684 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:38,684 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/B, priority=12, startTime=1732084958260; duration=0sec 2024-11-20T06:42:38,684 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:38,684 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:B 2024-11-20T06:42:38,684 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T06:42:38,685 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T06:42:38,685 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/C is initiating minor compaction (all files) 2024-11-20T06:42:38,685 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/C in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:38,685 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/37b0c1709a1f4e5d8c8f927782999d12, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/81d49a1a7ed148e595e55af97425009b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/aa59d273508e45908ba2dcc60d0b7e42, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/1cc33171163447c58b82b7202042c953] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=48.1 K 2024-11-20T06:42:38,685 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b0c1709a1f4e5d8c8f927782999d12, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732084952379 2024-11-20T06:42:38,685 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d49a1a7ed148e595e55af97425009b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732084952761 2024-11-20T06:42:38,686 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting aa59d273508e45908ba2dcc60d0b7e42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732084953955 2024-11-20T06:42:38,686 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cc33171163447c58b82b7202042c953, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732084955438 2024-11-20T06:42:38,689 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#A#compaction#511 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:38,689 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/1bd617bcd58a408ea8252988daeb1f6c is 175, key is test_row_0/A:col10/1732084955438/Put/seqid=0 2024-11-20T06:42:38,693 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#C#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:38,693 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a9e5e9b4b74f401f9703f0d79c3f789b is 50, key is test_row_0/C:col10/1732084955438/Put/seqid=0 2024-11-20T06:42:38,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742426_1602 (size=31903) 2024-11-20T06:42:38,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742427_1603 (size=12949) 2024-11-20T06:42:38,698 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/1bd617bcd58a408ea8252988daeb1f6c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/1bd617bcd58a408ea8252988daeb1f6c 2024-11-20T06:42:38,700 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a9e5e9b4b74f401f9703f0d79c3f789b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a9e5e9b4b74f401f9703f0d79c3f789b 2024-11-20T06:42:38,706 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/A of 03bb109fa60f64b2fccb76b2907dd474 into 1bd617bcd58a408ea8252988daeb1f6c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:38,706 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/C of 03bb109fa60f64b2fccb76b2907dd474 into a9e5e9b4b74f401f9703f0d79c3f789b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:38,706 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:38,706 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:38,706 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/C, priority=12, startTime=1732084958260; duration=0sec 2024-11-20T06:42:38,706 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/A, priority=12, startTime=1732084958260; duration=0sec 2024-11-20T06:42:38,706 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:38,706 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:38,706 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:A 2024-11-20T06:42:38,706 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:C 2024-11-20T06:42:38,773 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/0c0e43e00710460d9a4cee31e9424e24 2024-11-20T06:42:38,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/9cf2d5665c1c4aadb39456b4c73dc71c is 50, key is test_row_0/B:col10/1732084956648/Put/seqid=0 2024-11-20T06:42:38,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742428_1604 (size=12301) 2024-11-20T06:42:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:38,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:38,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085018869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085018984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:39,184 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/9cf2d5665c1c4aadb39456b4c73dc71c 2024-11-20T06:42:39,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/b4fd1e7488ed43ab9dcbc27cc01ab2ac is 50, key is test_row_0/C:col10/1732084956648/Put/seqid=0 2024-11-20T06:42:39,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742429_1605 (size=12301) 2024-11-20T06:42:39,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085019193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:39,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085019504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:39,592 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/b4fd1e7488ed43ab9dcbc27cc01ab2ac 2024-11-20T06:42:39,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/0c0e43e00710460d9a4cee31e9424e24 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/0c0e43e00710460d9a4cee31e9424e24 2024-11-20T06:42:39,598 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/0c0e43e00710460d9a4cee31e9424e24, entries=150, sequenceid=289, filesize=30.5 K 2024-11-20T06:42:39,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/9cf2d5665c1c4aadb39456b4c73dc71c as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9cf2d5665c1c4aadb39456b4c73dc71c 2024-11-20T06:42:39,601 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9cf2d5665c1c4aadb39456b4c73dc71c, entries=150, sequenceid=289, filesize=12.0 K 2024-11-20T06:42:39,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/b4fd1e7488ed43ab9dcbc27cc01ab2ac as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b4fd1e7488ed43ab9dcbc27cc01ab2ac 2024-11-20T06:42:39,604 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b4fd1e7488ed43ab9dcbc27cc01ab2ac, entries=150, sequenceid=289, filesize=12.0 K 2024-11-20T06:42:39,605 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 03bb109fa60f64b2fccb76b2907dd474 in 1249ms, sequenceid=289, compaction requested=false 2024-11-20T06:42:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:39,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-20T06:42:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-20T06:42:39,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-20T06:42:39,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0800 sec 2024-11-20T06:42:39,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.0840 sec 2024-11-20T06:42:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:40,015 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-20T06:42:40,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-20T06:42:40,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120996fd041dd3b4ddcba1e2541e9eb0a21_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084958843/Put/seqid=0 2024-11-20T06:42:40,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742430_1606 (size=14994) 2024-11-20T06:42:40,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085020057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:40,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085020163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:40,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085020374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:40,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:40,426 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:40,460 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120996fd041dd3b4ddcba1e2541e9eb0a21_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120996fd041dd3b4ddcba1e2541e9eb0a21_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:40,460 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/04fac72d6aec4ed98d1ab93a9f860d1b, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:40,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/04fac72d6aec4ed98d1ab93a9f860d1b is 175, key is test_row_0/A:col10/1732084958843/Put/seqid=0 2024-11-20T06:42:40,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742431_1607 (size=39949) 2024-11-20T06:42:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085020683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:40,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T06:42:40,694 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:40,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-20T06:42:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-20T06:42:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T06:42:40,699 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T06:42:40,700 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T06:42:40,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T06:42:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T06:42:40,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T06:42:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:40,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:40,864 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/04fac72d6aec4ed98d1ab93a9f860d1b 2024-11-20T06:42:40,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/5d9835e97367453f962b41154d4f71d1 is 50, key is test_row_0/B:col10/1732084958843/Put/seqid=0 2024-11-20T06:42:40,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742432_1608 (size=12301) 2024-11-20T06:42:41,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T06:42:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T06:42:41,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T06:42:41,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-20T06:42:41,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33847 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43432 deadline: 1732085021194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 , the old value is region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T06:42:41,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., hostname=61c2066a8a0f,33847,1732084794617, seqNum=5 because the exception is null or not the one we care about 2024-11-20T06:42:41,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/5d9835e97367453f962b41154d4f71d1 2024-11-20T06:42:41,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/5fa4bfee7f4c4fe99ef11496ff8f9182 is 50, key is test_row_0/C:col10/1732084958843/Put/seqid=0 2024-11-20T06:42:41,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742433_1609 (size=12301) 2024-11-20T06:42:41,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T06:42:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T06:42:41,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T06:42:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T06:42:41,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:41,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:42:41,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/5fa4bfee7f4c4fe99ef11496ff8f9182 2024-11-20T06:42:41,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/04fac72d6aec4ed98d1ab93a9f860d1b as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/04fac72d6aec4ed98d1ab93a9f860d1b 2024-11-20T06:42:41,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/04fac72d6aec4ed98d1ab93a9f860d1b, entries=200, sequenceid=314, filesize=39.0 K 2024-11-20T06:42:41,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/5d9835e97367453f962b41154d4f71d1 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/5d9835e97367453f962b41154d4f71d1 2024-11-20T06:42:41,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/5d9835e97367453f962b41154d4f71d1, entries=150, sequenceid=314, filesize=12.0 K 2024-11-20T06:42:41,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/5fa4bfee7f4c4fe99ef11496ff8f9182 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5fa4bfee7f4c4fe99ef11496ff8f9182 2024-11-20T06:42:41,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5fa4bfee7f4c4fe99ef11496ff8f9182, entries=150, sequenceid=314, filesize=12.0 K 2024-11-20T06:42:41,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 03bb109fa60f64b2fccb76b2907dd474 in 1694ms, sequenceid=314, compaction requested=true 2024-11-20T06:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T06:42:41,709 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T06:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:41,709 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03bb109fa60f64b2fccb76b2907dd474:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T06:42:41,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/B is initiating minor compaction (all files) 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/A is initiating minor compaction (all files) 2024-11-20T06:42:41,710 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/A in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,710 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/B in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,710 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/1bd617bcd58a408ea8252988daeb1f6c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/0c0e43e00710460d9a4cee31e9424e24, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/04fac72d6aec4ed98d1ab93a9f860d1b] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=100.7 K 2024-11-20T06:42:41,710 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9729dbca00b34364a345ab5fb4bbf9ad, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9cf2d5665c1c4aadb39456b4c73dc71c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/5d9835e97367453f962b41154d4f71d1] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=36.7 K 2024-11-20T06:42:41,710 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. files: [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/1bd617bcd58a408ea8252988daeb1f6c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/0c0e43e00710460d9a4cee31e9424e24, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/04fac72d6aec4ed98d1ab93a9f860d1b] 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 9729dbca00b34364a345ab5fb4bbf9ad, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732084955438 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bd617bcd58a408ea8252988daeb1f6c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732084955438 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf2d5665c1c4aadb39456b4c73dc71c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732084956636 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c0e43e00710460d9a4cee31e9424e24, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732084956636 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d9835e97367453f962b41154d4f71d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732084958843 2024-11-20T06:42:41,710 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04fac72d6aec4ed98d1ab93a9f860d1b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732084958843 2024-11-20T06:42:41,714 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:41,716 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:41,716 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/6597694f16744f80b880b5ccefa3bf02 is 50, key is test_row_0/B:col10/1732084958843/Put/seqid=0 2024-11-20T06:42:41,717 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120c80c189723c340adacf3fabd3cfa9a16_03bb109fa60f64b2fccb76b2907dd474 store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:41,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742434_1610 (size=13051) 2024-11-20T06:42:41,719 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120c80c189723c340adacf3fabd3cfa9a16_03bb109fa60f64b2fccb76b2907dd474, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:41,719 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c80c189723c340adacf3fabd3cfa9a16_03bb109fa60f64b2fccb76b2907dd474 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:41,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742435_1611 (size=4469) 2024-11-20T06:42:41,724 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#A#compaction#520 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:41,725 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/8dca86c4f77c482bb5ea723cc9496269 is 175, key is test_row_0/A:col10/1732084958843/Put/seqid=0 2024-11-20T06:42:41,730 INFO [Thread-2336 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2336. 2024-11-20T06:42:41,730 INFO [Thread-2344 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2344. 2024-11-20T06:42:41,730 DEBUG [Thread-2336 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:41,730 DEBUG [Thread-2344 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:41,730 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:41,730 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:41,731 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:41,731 DEBUG [Thread-2344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:41,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:41,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:41,731 INFO [Thread-2338 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2338. 2024-11-20T06:42:41,731 DEBUG [Thread-2338 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:41,731 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:41,731 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:41,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:41,733 INFO [Thread-2342 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2342. 2024-11-20T06:42:41,733 DEBUG [Thread-2342 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:41,733 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:41,733 DEBUG [Thread-2342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:41,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:41,734 INFO [Thread-2340 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2340. 2024-11-20T06:42:41,734 DEBUG [Thread-2340 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:41,734 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:41,734 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:41,734 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:41,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742436_1612 (size=32005) 2024-11-20T06:42:41,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T06:42:41,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:41,768 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-20T06:42:41,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200a7a3ff0bfd445aeacbc5346d43dbc61_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084960033/Put/seqid=0 2024-11-20T06:42:41,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742437_1613 (size=12454) 2024-11-20T06:42:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T06:42:42,125 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/6597694f16744f80b880b5ccefa3bf02 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/6597694f16744f80b880b5ccefa3bf02 2024-11-20T06:42:42,131 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/B of 03bb109fa60f64b2fccb76b2907dd474 into 6597694f16744f80b880b5ccefa3bf02(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:42,131 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:42,131 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/B, priority=13, startTime=1732084961709; duration=0sec 2024-11-20T06:42:42,132 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T06:42:42,132 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:B 2024-11-20T06:42:42,132 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T06:42:42,133 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T06:42:42,133 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1541): 03bb109fa60f64b2fccb76b2907dd474/C is initiating minor compaction (all files) 2024-11-20T06:42:42,133 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03bb109fa60f64b2fccb76b2907dd474/C in TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:42,133 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a9e5e9b4b74f401f9703f0d79c3f789b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b4fd1e7488ed43ab9dcbc27cc01ab2ac, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5fa4bfee7f4c4fe99ef11496ff8f9182] into tmpdir=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp, totalSize=36.7 K 2024-11-20T06:42:42,134 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting a9e5e9b4b74f401f9703f0d79c3f789b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732084955438 2024-11-20T06:42:42,134 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting b4fd1e7488ed43ab9dcbc27cc01ab2ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732084956636 2024-11-20T06:42:42,134 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fa4bfee7f4c4fe99ef11496ff8f9182, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732084958843 2024-11-20T06:42:42,141 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/8dca86c4f77c482bb5ea723cc9496269 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/8dca86c4f77c482bb5ea723cc9496269 2024-11-20T06:42:42,142 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03bb109fa60f64b2fccb76b2907dd474#C#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T06:42:42,142 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/946431b2b77d4969b500076631e1d5ea is 50, key is test_row_0/C:col10/1732084958843/Put/seqid=0 2024-11-20T06:42:42,145 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/A of 03bb109fa60f64b2fccb76b2907dd474 into 8dca86c4f77c482bb5ea723cc9496269(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:42,145 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:42,145 INFO [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/A, priority=13, startTime=1732084961709; duration=0sec 2024-11-20T06:42:42,145 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:42,145 DEBUG [RS:0;61c2066a8a0f:33847-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:A 2024-11-20T06:42:42,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742438_1614 (size=13051) 2024-11-20T06:42:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33847 {}] regionserver.HRegion(8855): Flush requested on 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:42,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. as already flushing 2024-11-20T06:42:42,176 INFO [Thread-2329 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2329. 2024-11-20T06:42:42,176 DEBUG [Thread-2329 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:42,176 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:42,176 DEBUG [Thread-2329 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:42,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:42,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:42,192 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200a7a3ff0bfd445aeacbc5346d43dbc61_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200a7a3ff0bfd445aeacbc5346d43dbc61_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/f83040ba957241fea736e0e6ed8577fe, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:42,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/f83040ba957241fea736e0e6ed8577fe is 175, key is test_row_0/A:col10/1732084960033/Put/seqid=0 2024-11-20T06:42:42,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742439_1615 (size=31255) 2024-11-20T06:42:42,206 INFO [Thread-2327 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2327. 2024-11-20T06:42:42,206 DEBUG [Thread-2327 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:42,206 DEBUG [Thread-2327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:42,206 DEBUG [Thread-2327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:42,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:42,551 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/946431b2b77d4969b500076631e1d5ea as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/946431b2b77d4969b500076631e1d5ea 2024-11-20T06:42:42,557 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03bb109fa60f64b2fccb76b2907dd474/C of 03bb109fa60f64b2fccb76b2907dd474 into 946431b2b77d4969b500076631e1d5ea(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T06:42:42,557 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:42,557 INFO [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., storeName=03bb109fa60f64b2fccb76b2907dd474/C, priority=13, startTime=1732084961709; duration=0sec 2024-11-20T06:42:42,557 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T06:42:42,557 DEBUG [RS:0;61c2066a8a0f:33847-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03bb109fa60f64b2fccb76b2907dd474:C 2024-11-20T06:42:42,601 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/f83040ba957241fea736e0e6ed8577fe 2024-11-20T06:42:42,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/79c8ff1b69a74fc2b1e9ef6883101594 is 50, key is test_row_0/B:col10/1732084960033/Put/seqid=0 2024-11-20T06:42:42,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742440_1616 (size=12301) 2024-11-20T06:42:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T06:42:43,017 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/79c8ff1b69a74fc2b1e9ef6883101594 2024-11-20T06:42:43,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/957a739287cb4c68bf7a679a7ad6d6f7 is 50, key is test_row_0/C:col10/1732084960033/Put/seqid=0 2024-11-20T06:42:43,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742441_1617 (size=12301) 2024-11-20T06:42:43,436 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/957a739287cb4c68bf7a679a7ad6d6f7 2024-11-20T06:42:43,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/f83040ba957241fea736e0e6ed8577fe as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/f83040ba957241fea736e0e6ed8577fe 2024-11-20T06:42:43,447 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/f83040ba957241fea736e0e6ed8577fe, entries=150, sequenceid=328, filesize=30.5 K 2024-11-20T06:42:43,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/79c8ff1b69a74fc2b1e9ef6883101594 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/79c8ff1b69a74fc2b1e9ef6883101594 2024-11-20T06:42:43,453 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/79c8ff1b69a74fc2b1e9ef6883101594, entries=150, sequenceid=328, filesize=12.0 K 2024-11-20T06:42:43,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/957a739287cb4c68bf7a679a7ad6d6f7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/957a739287cb4c68bf7a679a7ad6d6f7 2024-11-20T06:42:43,458 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/957a739287cb4c68bf7a679a7ad6d6f7, entries=150, sequenceid=328, filesize=12.0 K 2024-11-20T06:42:43,459 INFO [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=13.42 KB/13740 for 03bb109fa60f64b2fccb76b2907dd474 in 1691ms, sequenceid=328, compaction requested=false 2024-11-20T06:42:43,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 03bb109fa60f64b2fccb76b2907dd474: 2024-11-20T06:42:43,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:43,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-20T06:42:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-20T06:42:43,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-20T06:42:43,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7600 sec 2024-11-20T06:42:43,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.7690 sec 2024-11-20T06:42:44,515 INFO [Thread-2333 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2333. 2024-11-20T06:42:44,515 DEBUG [Thread-2333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:44,515 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:44,515 DEBUG [Thread-2333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:44,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:44,525 INFO [Thread-2331 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2331. 2024-11-20T06:42:44,525 DEBUG [Thread-2331 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:44,525 DEBUG [Thread-2331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:44,526 DEBUG [Thread-2331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:44,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:44,536 INFO [Thread-2325 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2325. 2024-11-20T06:42:44,536 DEBUG [Thread-2325 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T06:42:44,537 DEBUG [Thread-2325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:44,537 DEBUG [Thread-2325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:44,537 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T06:42:44,874 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3821 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3759 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3811 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3815 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3777 2024-11-20T06:42:44,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T06:42:44,874 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:42:44,874 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:42:44,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:44,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:44,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:44,875 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-20T06:42:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T06:42:44,878 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084964878"}]},"ts":"1732084964878"} 2024-11-20T06:42:44,879 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T06:42:44,879 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T06:42:44,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T06:42:44,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, UNASSIGN}] 2024-11-20T06:42:44,881 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, UNASSIGN 2024-11-20T06:42:44,882 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=03bb109fa60f64b2fccb76b2907dd474, regionState=CLOSING, regionLocation=61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:44,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, UNASSIGN because future has completed 2024-11-20T06:42:44,883 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T06:42:44,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617}] 2024-11-20T06:42:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T06:42:45,035 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:45,035 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T06:42:45,035 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 03bb109fa60f64b2fccb76b2907dd474, disabling compactions & flushes 2024-11-20T06:42:45,035 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:45,035 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:45,035 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. after waiting 0 ms 2024-11-20T06:42:45,035 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:45,036 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 03bb109fa60f64b2fccb76b2907dd474 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-20T06:42:45,042 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112098e47e6c41d24f9bbcc8ca9450a5ecb1_03bb109fa60f64b2fccb76b2907dd474 is 50, key is test_row_0/A:col10/1732084964524/Put/seqid=0 2024-11-20T06:42:45,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742442_1618 (size=12454) 2024-11-20T06:42:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T06:42:45,447 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:42:45,455 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112098e47e6c41d24f9bbcc8ca9450a5ecb1_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112098e47e6c41d24f9bbcc8ca9450a5ecb1_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:45,457 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bb50eca620b244409032eeffa62062f9, store: [table=TestAcidGuarantees family=A region=03bb109fa60f64b2fccb76b2907dd474] 2024-11-20T06:42:45,457 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bb50eca620b244409032eeffa62062f9 is 175, key is test_row_0/A:col10/1732084964524/Put/seqid=0 2024-11-20T06:42:45,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742443_1619 (size=31255) 2024-11-20T06:42:45,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T06:42:45,862 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bb50eca620b244409032eeffa62062f9 2024-11-20T06:42:45,875 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/f9563db338a2441c8bc796b5a7bcf602 is 50, key is test_row_0/B:col10/1732084964524/Put/seqid=0 2024-11-20T06:42:45,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742444_1620 (size=12301) 2024-11-20T06:42:45,879 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/f9563db338a2441c8bc796b5a7bcf602 2024-11-20T06:42:45,885 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a4e8050c4825462882a2490a4d144a2f is 50, key is test_row_0/C:col10/1732084964524/Put/seqid=0 2024-11-20T06:42:45,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742445_1621 (size=12301) 2024-11-20T06:42:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T06:42:46,289 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a4e8050c4825462882a2490a4d144a2f 2024-11-20T06:42:46,297 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/A/bb50eca620b244409032eeffa62062f9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bb50eca620b244409032eeffa62062f9 2024-11-20T06:42:46,305 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bb50eca620b244409032eeffa62062f9, entries=150, sequenceid=339, filesize=30.5 K 2024-11-20T06:42:46,306 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/B/f9563db338a2441c8bc796b5a7bcf602 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/f9563db338a2441c8bc796b5a7bcf602 2024-11-20T06:42:46,310 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/f9563db338a2441c8bc796b5a7bcf602, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T06:42:46,311 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/.tmp/C/a4e8050c4825462882a2490a4d144a2f as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a4e8050c4825462882a2490a4d144a2f 2024-11-20T06:42:46,315 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a4e8050c4825462882a2490a4d144a2f, entries=150, sequenceid=339, filesize=12.0 K 2024-11-20T06:42:46,316 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 03bb109fa60f64b2fccb76b2907dd474 in 1281ms, sequenceid=339, compaction requested=true 2024-11-20T06:42:46,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/5ad44c8195cf43e78872ebb3cf82141b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/23a3e1b62d494ef9aff5ab10bca6c62d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/cb9fe7d029284f868de3a910e92c2df7, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/945f8965d015479ab454d5c8500b75ff, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/77f924a86f55465ab7f0023c8e566065, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bffc3eabcbf947b08b5f791dfd2e9129, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bb3c4fd8fd6e457e9fc46856c3253a36, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/a43e0c4d04e844428d01163759a2be67, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/4e7e0f766cd244ddbda1e08f381ad511, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/7476366812e34b98bcc26f1a3cf36c5d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/024ee345dc454d268636a50d0e02ccd6, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/d87c39436fcb4d2aaa7d2b19d0356444, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/252ea1249df540ff9005a6c6cce20710, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/846ca60c6bc2483fa77a885b348d03d3, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/de100b0dae80457082c25fc67f5775fd, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bc974041ca5d457dbfe0684f3080dcdc, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/b40aeeec431847afb98d4502acc35171, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/fbee1ae95ab3444abc5422dea4b6db8d, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/1bd617bcd58a408ea8252988daeb1f6c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/dda7326ac94a41498dd16c708262194b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/0c0e43e00710460d9a4cee31e9424e24, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/04fac72d6aec4ed98d1ab93a9f860d1b] to archive 2024-11-20T06:42:46,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:42:46,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/5ad44c8195cf43e78872ebb3cf82141b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/5ad44c8195cf43e78872ebb3cf82141b 2024-11-20T06:42:46,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/23a3e1b62d494ef9aff5ab10bca6c62d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/23a3e1b62d494ef9aff5ab10bca6c62d 2024-11-20T06:42:46,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/cb9fe7d029284f868de3a910e92c2df7 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/cb9fe7d029284f868de3a910e92c2df7 2024-11-20T06:42:46,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/945f8965d015479ab454d5c8500b75ff to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/945f8965d015479ab454d5c8500b75ff 2024-11-20T06:42:46,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/77f924a86f55465ab7f0023c8e566065 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/77f924a86f55465ab7f0023c8e566065 2024-11-20T06:42:46,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bffc3eabcbf947b08b5f791dfd2e9129 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bffc3eabcbf947b08b5f791dfd2e9129 2024-11-20T06:42:46,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bb3c4fd8fd6e457e9fc46856c3253a36 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bb3c4fd8fd6e457e9fc46856c3253a36 2024-11-20T06:42:46,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/a43e0c4d04e844428d01163759a2be67 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/a43e0c4d04e844428d01163759a2be67 2024-11-20T06:42:46,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/4e7e0f766cd244ddbda1e08f381ad511 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/4e7e0f766cd244ddbda1e08f381ad511 2024-11-20T06:42:46,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/7476366812e34b98bcc26f1a3cf36c5d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/7476366812e34b98bcc26f1a3cf36c5d 2024-11-20T06:42:46,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/024ee345dc454d268636a50d0e02ccd6 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/024ee345dc454d268636a50d0e02ccd6 2024-11-20T06:42:46,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/d87c39436fcb4d2aaa7d2b19d0356444 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/d87c39436fcb4d2aaa7d2b19d0356444 2024-11-20T06:42:46,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/252ea1249df540ff9005a6c6cce20710 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/252ea1249df540ff9005a6c6cce20710 2024-11-20T06:42:46,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/846ca60c6bc2483fa77a885b348d03d3 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/846ca60c6bc2483fa77a885b348d03d3 2024-11-20T06:42:46,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/de100b0dae80457082c25fc67f5775fd to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/de100b0dae80457082c25fc67f5775fd 2024-11-20T06:42:46,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bc974041ca5d457dbfe0684f3080dcdc to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bc974041ca5d457dbfe0684f3080dcdc 2024-11-20T06:42:46,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/b40aeeec431847afb98d4502acc35171 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/b40aeeec431847afb98d4502acc35171 2024-11-20T06:42:46,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/fbee1ae95ab3444abc5422dea4b6db8d to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/fbee1ae95ab3444abc5422dea4b6db8d 2024-11-20T06:42:46,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/1bd617bcd58a408ea8252988daeb1f6c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/1bd617bcd58a408ea8252988daeb1f6c 2024-11-20T06:42:46,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/dda7326ac94a41498dd16c708262194b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/dda7326ac94a41498dd16c708262194b 2024-11-20T06:42:46,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/0c0e43e00710460d9a4cee31e9424e24 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/0c0e43e00710460d9a4cee31e9424e24 2024-11-20T06:42:46,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/04fac72d6aec4ed98d1ab93a9f860d1b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/04fac72d6aec4ed98d1ab93a9f860d1b 2024-11-20T06:42:46,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/4751771786d0483fa49df93f5d085acc, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/394a6ef5621a4c729f63756eaba98653, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/c1eea16ce2244daebf64ffe32cbe9736, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d6409f137c94482c8d405497e7021e77, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/27d6f1f0b6214f229e3f244f01498248, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/94748e993ac148f299eb45468baa9a4e, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/97537c9c02f3417db4bf4f47c1650136, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/792196b159cb45ef96b3692e14bdd70b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/3598cc27bf81442faa63558bb8e5978f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2dffa7d558364d7194713a22b605c81a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d9c071e487834bcf8b6d9ddcc63b4b18, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/29f8243e09a94d7ca791c7d6be88323a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/7e7c4339ade74e81aea53a1e94c79a0f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/f2a353945b754f9ba3022d1604b9e7f4, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2343501f78044f56919998df60495a7a, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/ed5e1e42e8154f48a63253d95b9d83e2, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/0f7f5f9260364583b11d377cff0b24ae, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/73c5a2d0095e4ea9b4ac2b427201f78f, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9729dbca00b34364a345ab5fb4bbf9ad, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/a9a652e0e17244b6af9ad7d21ae1c6df, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9cf2d5665c1c4aadb39456b4c73dc71c, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/5d9835e97367453f962b41154d4f71d1] to archive 2024-11-20T06:42:46,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:42:46,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/4751771786d0483fa49df93f5d085acc to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/4751771786d0483fa49df93f5d085acc 2024-11-20T06:42:46,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/394a6ef5621a4c729f63756eaba98653 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/394a6ef5621a4c729f63756eaba98653 2024-11-20T06:42:46,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/c1eea16ce2244daebf64ffe32cbe9736 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/c1eea16ce2244daebf64ffe32cbe9736 2024-11-20T06:42:46,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d6409f137c94482c8d405497e7021e77 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d6409f137c94482c8d405497e7021e77 2024-11-20T06:42:46,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/27d6f1f0b6214f229e3f244f01498248 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/27d6f1f0b6214f229e3f244f01498248 2024-11-20T06:42:46,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/94748e993ac148f299eb45468baa9a4e to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/94748e993ac148f299eb45468baa9a4e 2024-11-20T06:42:46,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/97537c9c02f3417db4bf4f47c1650136 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/97537c9c02f3417db4bf4f47c1650136 2024-11-20T06:42:46,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/792196b159cb45ef96b3692e14bdd70b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/792196b159cb45ef96b3692e14bdd70b 2024-11-20T06:42:46,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/3598cc27bf81442faa63558bb8e5978f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/3598cc27bf81442faa63558bb8e5978f 2024-11-20T06:42:46,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2dffa7d558364d7194713a22b605c81a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2dffa7d558364d7194713a22b605c81a 2024-11-20T06:42:46,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d9c071e487834bcf8b6d9ddcc63b4b18 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/d9c071e487834bcf8b6d9ddcc63b4b18 2024-11-20T06:42:46,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/29f8243e09a94d7ca791c7d6be88323a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/29f8243e09a94d7ca791c7d6be88323a 2024-11-20T06:42:46,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/7e7c4339ade74e81aea53a1e94c79a0f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/7e7c4339ade74e81aea53a1e94c79a0f 2024-11-20T06:42:46,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/f2a353945b754f9ba3022d1604b9e7f4 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/f2a353945b754f9ba3022d1604b9e7f4 2024-11-20T06:42:46,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2343501f78044f56919998df60495a7a to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/2343501f78044f56919998df60495a7a 2024-11-20T06:42:46,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/ed5e1e42e8154f48a63253d95b9d83e2 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/ed5e1e42e8154f48a63253d95b9d83e2 2024-11-20T06:42:46,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/0f7f5f9260364583b11d377cff0b24ae to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/0f7f5f9260364583b11d377cff0b24ae 2024-11-20T06:42:46,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/73c5a2d0095e4ea9b4ac2b427201f78f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/73c5a2d0095e4ea9b4ac2b427201f78f 2024-11-20T06:42:46,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9729dbca00b34364a345ab5fb4bbf9ad to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9729dbca00b34364a345ab5fb4bbf9ad 2024-11-20T06:42:46,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/a9a652e0e17244b6af9ad7d21ae1c6df to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/a9a652e0e17244b6af9ad7d21ae1c6df 2024-11-20T06:42:46,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9cf2d5665c1c4aadb39456b4c73dc71c to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/9cf2d5665c1c4aadb39456b4c73dc71c 2024-11-20T06:42:46,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/5d9835e97367453f962b41154d4f71d1 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/5d9835e97367453f962b41154d4f71d1 2024-11-20T06:42:46,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/0d22a543f9d94dc6a2444ae801fa4bc1, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/90cb0fa3d0584ba1b3878c83cd297f07, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/bb166f21124644c1b61d5925f2d6f246, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f7ed59fe4a47bbb153b1299ae9ac11, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b23878f2c3b94a0cbf36b78ff1d6a858, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3b96b8e9f0344badbccf5c3ed5cff5ba, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/4495ccab2ffe45caa464b91e867f3089, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5f35f129464c4d27972a6679fa6893df, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/47394d73fcdd4d80a1ae1fefbc609658, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b008fb7b7cef478993e15d77f8cc4eeb, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5575a7a3599f4151ad7428eb334b8b11, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/d4f5d325201d44f7ab1af24372524f71, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/f84d60e9f3914cff9d4c71e23281dc85, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f579e6fa30453d846d714e50373e49, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/37b0c1709a1f4e5d8c8f927782999d12, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3c55acd6b24d4cd383a68d073a5ca4ab, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/81d49a1a7ed148e595e55af97425009b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/aa59d273508e45908ba2dcc60d0b7e42, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a9e5e9b4b74f401f9703f0d79c3f789b, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/1cc33171163447c58b82b7202042c953, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b4fd1e7488ed43ab9dcbc27cc01ab2ac, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5fa4bfee7f4c4fe99ef11496ff8f9182] to archive 2024-11-20T06:42:46,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T06:42:46,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/0d22a543f9d94dc6a2444ae801fa4bc1 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/0d22a543f9d94dc6a2444ae801fa4bc1 2024-11-20T06:42:46,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/90cb0fa3d0584ba1b3878c83cd297f07 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/90cb0fa3d0584ba1b3878c83cd297f07 2024-11-20T06:42:46,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/bb166f21124644c1b61d5925f2d6f246 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/bb166f21124644c1b61d5925f2d6f246 2024-11-20T06:42:46,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f7ed59fe4a47bbb153b1299ae9ac11 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f7ed59fe4a47bbb153b1299ae9ac11 2024-11-20T06:42:46,359 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b23878f2c3b94a0cbf36b78ff1d6a858 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b23878f2c3b94a0cbf36b78ff1d6a858 2024-11-20T06:42:46,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3b96b8e9f0344badbccf5c3ed5cff5ba to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3b96b8e9f0344badbccf5c3ed5cff5ba 2024-11-20T06:42:46,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/4495ccab2ffe45caa464b91e867f3089 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/4495ccab2ffe45caa464b91e867f3089 2024-11-20T06:42:46,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5f35f129464c4d27972a6679fa6893df to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5f35f129464c4d27972a6679fa6893df 2024-11-20T06:42:46,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/47394d73fcdd4d80a1ae1fefbc609658 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/47394d73fcdd4d80a1ae1fefbc609658 2024-11-20T06:42:46,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b008fb7b7cef478993e15d77f8cc4eeb to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b008fb7b7cef478993e15d77f8cc4eeb 2024-11-20T06:42:46,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5575a7a3599f4151ad7428eb334b8b11 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5575a7a3599f4151ad7428eb334b8b11 2024-11-20T06:42:46,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/d4f5d325201d44f7ab1af24372524f71 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/d4f5d325201d44f7ab1af24372524f71 2024-11-20T06:42:46,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/f84d60e9f3914cff9d4c71e23281dc85 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/f84d60e9f3914cff9d4c71e23281dc85 2024-11-20T06:42:46,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f579e6fa30453d846d714e50373e49 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a2f579e6fa30453d846d714e50373e49 2024-11-20T06:42:46,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/37b0c1709a1f4e5d8c8f927782999d12 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/37b0c1709a1f4e5d8c8f927782999d12 2024-11-20T06:42:46,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3c55acd6b24d4cd383a68d073a5ca4ab to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/3c55acd6b24d4cd383a68d073a5ca4ab 2024-11-20T06:42:46,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/81d49a1a7ed148e595e55af97425009b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/81d49a1a7ed148e595e55af97425009b 2024-11-20T06:42:46,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/aa59d273508e45908ba2dcc60d0b7e42 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/aa59d273508e45908ba2dcc60d0b7e42 2024-11-20T06:42:46,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a9e5e9b4b74f401f9703f0d79c3f789b to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a9e5e9b4b74f401f9703f0d79c3f789b 2024-11-20T06:42:46,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/1cc33171163447c58b82b7202042c953 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/1cc33171163447c58b82b7202042c953 2024-11-20T06:42:46,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b4fd1e7488ed43ab9dcbc27cc01ab2ac to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/b4fd1e7488ed43ab9dcbc27cc01ab2ac 2024-11-20T06:42:46,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5fa4bfee7f4c4fe99ef11496ff8f9182 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/5fa4bfee7f4c4fe99ef11496ff8f9182 2024-11-20T06:42:46,373 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/recovered.edits/342.seqid, newMaxSeqId=342, maxSeqId=4 2024-11-20T06:42:46,373 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. 2024-11-20T06:42:46,373 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 03bb109fa60f64b2fccb76b2907dd474: Waiting for close lock at 1732084965035Running coprocessor pre-close hooks at 1732084965035Disabling compacts and flushes for region at 1732084965035Disabling writes for close at 1732084965035Obtaining lock to block concurrent updates at 1732084965036 (+1 ms)Preparing flush snapshotting stores in 03bb109fa60f64b2fccb76b2907dd474 at 1732084965036Finished memstore snapshotting TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732084965036Flushing stores of TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474. at 1732084965036Flushing 03bb109fa60f64b2fccb76b2907dd474/A: creating writer at 1732084965036Flushing mob file 03bb109fa60f64b2fccb76b2907dd474/A: appending metadata at 1732084965042 (+6 ms)Flushing mob file 03bb109fa60f64b2fccb76b2907dd474/A: closing flushed file at 1732084965042Flushing 03bb109fa60f64b2fccb76b2907dd474/A: appending metadata at 1732084965457 (+415 ms)Flushing 03bb109fa60f64b2fccb76b2907dd474/A: closing flushed file at 1732084965457Flushing 03bb109fa60f64b2fccb76b2907dd474/B: creating writer at 1732084965871 (+414 ms)Flushing 03bb109fa60f64b2fccb76b2907dd474/B: appending metadata at 1732084965875 (+4 ms)Flushing 03bb109fa60f64b2fccb76b2907dd474/B: closing flushed file at 1732084965875Flushing 03bb109fa60f64b2fccb76b2907dd474/C: creating writer at 1732084965882 (+7 ms)Flushing 03bb109fa60f64b2fccb76b2907dd474/C: appending metadata at 1732084965885 (+3 ms)Flushing 03bb109fa60f64b2fccb76b2907dd474/C: closing flushed file at 1732084965885Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e1e66dc: reopening flushed file at 1732084966296 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24afe4e1: reopening flushed file at 1732084966305 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39ecfe00: reopening flushed file at 1732084966310 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 03bb109fa60f64b2fccb76b2907dd474 in 1281ms, sequenceid=339, compaction requested=true at 1732084966316 (+6 ms)Writing region close event to WAL at 1732084966370 (+54 ms)Running coprocessor post-close hooks at 1732084966373 (+3 ms)Closed at 1732084966373 2024-11-20T06:42:46,375 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:46,375 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=03bb109fa60f64b2fccb76b2907dd474, regionState=CLOSED 2024-11-20T06:42:46,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 because future has completed 2024-11-20T06:42:46,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-20T06:42:46,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 03bb109fa60f64b2fccb76b2907dd474, server=61c2066a8a0f,33847,1732084794617 in 1.4940 sec 2024-11-20T06:42:46,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-20T06:42:46,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=03bb109fa60f64b2fccb76b2907dd474, UNASSIGN in 1.4980 sec 2024-11-20T06:42:46,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-20T06:42:46,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5020 sec 2024-11-20T06:42:46,384 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732084966384"}]},"ts":"1732084966384"} 2024-11-20T06:42:46,385 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T06:42:46,385 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T06:42:46,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5110 sec 2024-11-20T06:42:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T06:42:47,026 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:47,027 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-20T06:42:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:47,033 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T06:42:47,035 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:47,040 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,043 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C, FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/recovered.edits] 2024-11-20T06:42:47,045 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/8dca86c4f77c482bb5ea723cc9496269 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/8dca86c4f77c482bb5ea723cc9496269 2024-11-20T06:42:47,046 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bb50eca620b244409032eeffa62062f9 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/bb50eca620b244409032eeffa62062f9 2024-11-20T06:42:47,047 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/f83040ba957241fea736e0e6ed8577fe to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/A/f83040ba957241fea736e0e6ed8577fe 2024-11-20T06:42:47,049 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/6597694f16744f80b880b5ccefa3bf02 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/6597694f16744f80b880b5ccefa3bf02 2024-11-20T06:42:47,050 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/79c8ff1b69a74fc2b1e9ef6883101594 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/79c8ff1b69a74fc2b1e9ef6883101594 2024-11-20T06:42:47,052 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/f9563db338a2441c8bc796b5a7bcf602 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/B/f9563db338a2441c8bc796b5a7bcf602 2024-11-20T06:42:47,054 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/946431b2b77d4969b500076631e1d5ea to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/946431b2b77d4969b500076631e1d5ea 2024-11-20T06:42:47,056 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/957a739287cb4c68bf7a679a7ad6d6f7 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/957a739287cb4c68bf7a679a7ad6d6f7 2024-11-20T06:42:47,058 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a4e8050c4825462882a2490a4d144a2f to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/C/a4e8050c4825462882a2490a4d144a2f 2024-11-20T06:42:47,060 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/recovered.edits/342.seqid to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474/recovered.edits/342.seqid 2024-11-20T06:42:47,061 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/default/TestAcidGuarantees/03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,061 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T06:42:47,061 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T06:42:47,062 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T06:42:47,064 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200a7a3ff0bfd445aeacbc5346d43dbc61_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200a7a3ff0bfd445aeacbc5346d43dbc61_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,065 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201126b8def5be40759602890e9912fd7e_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201126b8def5be40759602890e9912fd7e_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,066 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201d8d0743c7604713b60aed330357f949_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201d8d0743c7604713b60aed330357f949_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,066 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202b3a9e94d0bc4c8298b147b67f8d4d06_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202b3a9e94d0bc4c8298b147b67f8d4d06_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,067 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205c34a911a840403ab453cb782cfe6d61_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205c34a911a840403ab453cb782cfe6d61_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,068 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c73c4d8a9f241f39c1d32d7475dfc6b_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c73c4d8a9f241f39c1d32d7475dfc6b_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,068 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207d9d3c868fe442ff9efa481e580c5c9d_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207d9d3c868fe442ff9efa481e580c5c9d_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,069 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112098e47e6c41d24f9bbcc8ca9450a5ecb1_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112098e47e6c41d24f9bbcc8ca9450a5ecb1_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,069 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120996fd041dd3b4ddcba1e2541e9eb0a21_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120996fd041dd3b4ddcba1e2541e9eb0a21_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,070 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209c7d69d4410648249aa3633d7ee24636_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209c7d69d4410648249aa3633d7ee24636_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,071 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e31991e6f134ccbae94714c93964c15_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e31991e6f134ccbae94714c93964c15_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,071 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b2d5a6d51b004728b08a99701f820e9a_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b2d5a6d51b004728b08a99701f820e9a_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,072 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c06a6c72b51141678df7a3b4713a204e_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c06a6c72b51141678df7a3b4713a204e_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,073 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c7d113ff1ea64c5e85976436225948e1_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c7d113ff1ea64c5e85976436225948e1_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,073 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dde6168b72b44339a899d4c3046c581b_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dde6168b72b44339a899d4c3046c581b_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,074 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120deff2ef9e46d47e0b8e03a3298977a8b_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120deff2ef9e46d47e0b8e03a3298977a8b_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,075 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e4f31358749746f8803a195bc7447105_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e4f31358749746f8803a195bc7447105_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,075 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e8197295871f48baa8df2516d877277c_03bb109fa60f64b2fccb76b2907dd474 to hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e8197295871f48baa8df2516d877277c_03bb109fa60f64b2fccb76b2907dd474 2024-11-20T06:42:47,076 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T06:42:47,077 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:47,079 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T06:42:47,081 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T06:42:47,081 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:47,081 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T06:42:47,082 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732084967081"}]},"ts":"9223372036854775807"} 2024-11-20T06:42:47,083 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T06:42:47,083 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 03bb109fa60f64b2fccb76b2907dd474, NAME => 'TestAcidGuarantees,,1732084938597.03bb109fa60f64b2fccb76b2907dd474.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T06:42:47,083 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T06:42:47,083 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732084967083"}]},"ts":"9223372036854775807"} 2024-11-20T06:42:47,084 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T06:42:47,085 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T06:42:47,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-20T06:42:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33659 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T06:42:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T06:42:47,147 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T06:42:47,157 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 240), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=357 (was 361), ProcessCount=11 (was 11), AvailableMemoryMB=4076 (was 4403) 2024-11-20T06:42:47,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T06:42:47,158 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:42:47,158 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:42:47,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:47,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:47,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:42:47,158 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T06:42:47,158 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=958468940, stopped=false 2024-11-20T06:42:47,158 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c2066a8a0f,33659,1732084793752 2024-11-20T06:42:47,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T06:42:47,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T06:42:47,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:42:47,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:42:47,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T06:42:47,199 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:42:47,199 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:42:47,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:42:47,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:47,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:42:47,200 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,33847,1732084794617' ***** 2024-11-20T06:42:47,201 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T06:42:47,201 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T06:42:47,201 INFO [RS:0;61c2066a8a0f:33847 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T06:42:47,201 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T06:42:47,202 INFO [RS:0;61c2066a8a0f:33847 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T06:42:47,202 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:47,202 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T06:42:47,202 INFO [RS:0;61c2066a8a0f:33847 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c2066a8a0f:33847. 2024-11-20T06:42:47,202 DEBUG [RS:0;61c2066a8a0f:33847 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:42:47,202 DEBUG [RS:0;61c2066a8a0f:33847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:47,203 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T06:42:47,203 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T06:42:47,203 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T06:42:47,203 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T06:42:47,203 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T06:42:47,204 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T06:42:47,204 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T06:42:47,204 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T06:42:47,204 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T06:42:47,204 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T06:42:47,204 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T06:42:47,204 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T06:42:47,204 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-20T06:42:47,227 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/info/ba533b3eae2e4c2b8fcc39b2bfdaca64 is 95, key is TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec./info:/1732084826055/DeleteFamily/seqid=0 2024-11-20T06:42:47,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742446_1622 (size=6514) 2024-11-20T06:42:47,257 INFO [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:42:47,404 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T06:42:47,604 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T06:42:47,631 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/info/ba533b3eae2e4c2b8fcc39b2bfdaca64 2024-11-20T06:42:47,658 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/ns/f6d306baaa4b46a4a83444fd0cc4c2f4 is 93, key is TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec./ns:/1732084826005/DeleteFamily/seqid=0 2024-11-20T06:42:47,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742447_1623 (size=5906) 2024-11-20T06:42:47,764 INFO [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T06:42:47,764 INFO [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T06:42:47,805 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T06:42:48,005 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T06:42:48,063 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/ns/f6d306baaa4b46a4a83444fd0cc4c2f4 2024-11-20T06:42:48,093 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/rep_barrier/46d71308abb1472b915a89678c950574 is 102, key is TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec./rep_barrier:/1732084826005/DeleteFamily/seqid=0 2024-11-20T06:42:48,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742448_1624 (size=6025) 2024-11-20T06:42:48,205 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T06:42:48,406 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T06:42:48,498 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/rep_barrier/46d71308abb1472b915a89678c950574 2024-11-20T06:42:48,524 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/table/964e5466dd6b422fa068c4a7abedfe64 is 96, key is TestAcidGuarantees,,1732084799359.8b50cf3b5c22a8ba480f30fa5ecd19ec./table:/1732084826005/DeleteFamily/seqid=0 2024-11-20T06:42:48,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742449_1625 (size=5936) 2024-11-20T06:42:48,606 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T06:42:48,806 DEBUG [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T06:42:48,929 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/table/964e5466dd6b422fa068c4a7abedfe64 2024-11-20T06:42:48,938 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/info/ba533b3eae2e4c2b8fcc39b2bfdaca64 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/info/ba533b3eae2e4c2b8fcc39b2bfdaca64 2024-11-20T06:42:48,942 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/info/ba533b3eae2e4c2b8fcc39b2bfdaca64, entries=12, sequenceid=90, filesize=6.4 K 2024-11-20T06:42:48,943 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/ns/f6d306baaa4b46a4a83444fd0cc4c2f4 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/ns/f6d306baaa4b46a4a83444fd0cc4c2f4 2024-11-20T06:42:48,948 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/ns/f6d306baaa4b46a4a83444fd0cc4c2f4, entries=8, sequenceid=90, filesize=5.8 K 2024-11-20T06:42:48,949 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/rep_barrier/46d71308abb1472b915a89678c950574 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/rep_barrier/46d71308abb1472b915a89678c950574 2024-11-20T06:42:48,953 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/rep_barrier/46d71308abb1472b915a89678c950574, entries=6, sequenceid=90, filesize=5.9 K 2024-11-20T06:42:48,953 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/.tmp/table/964e5466dd6b422fa068c4a7abedfe64 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/table/964e5466dd6b422fa068c4a7abedfe64 2024-11-20T06:42:48,956 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/table/964e5466dd6b422fa068c4a7abedfe64, entries=7, sequenceid=90, filesize=5.8 K 2024-11-20T06:42:48,957 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1752ms, sequenceid=90, compaction requested=false 2024-11-20T06:42:48,963 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-20T06:42:48,963 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T06:42:48,963 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T06:42:48,963 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732084967204Running coprocessor pre-close hooks at 1732084967204Disabling compacts and flushes for region at 1732084967204Disabling writes for close at 1732084967204Obtaining lock to block concurrent updates at 1732084967204Preparing flush snapshotting stores in 1588230740 at 1732084967204Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732084967205 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732084967206 (+1 ms)Flushing 1588230740/info: creating writer at 1732084967206Flushing 1588230740/info: appending metadata at 1732084967227 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732084967227Flushing 1588230740/ns: creating writer at 1732084967636 (+409 ms)Flushing 1588230740/ns: appending metadata at 1732084967658 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1732084967658Flushing 1588230740/rep_barrier: creating writer at 1732084968072 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732084968093 (+21 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732084968093Flushing 1588230740/table: creating writer at 1732084968506 (+413 ms)Flushing 1588230740/table: appending metadata at 1732084968524 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732084968524Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67c81bf2: reopening flushed file at 1732084968937 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59243c7a: reopening flushed file at 1732084968943 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41638e0b: reopening flushed file at 1732084968948 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fd8e06e: reopening flushed file at 1732084968953 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1752ms, sequenceid=90, compaction requested=false at 1732084968957 (+4 ms)Writing region close event to WAL at 1732084968960 (+3 ms)Running coprocessor post-close hooks at 1732084968963 (+3 ms)Closed at 1732084968963 2024-11-20T06:42:48,963 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T06:42:49,007 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,33847,1732084794617; all regions closed. 2024-11-20T06:42:49,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741834_1010 (size=25344) 2024-11-20T06:42:49,016 DEBUG [RS:0;61c2066a8a0f:33847 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/oldWALs 2024-11-20T06:42:49,016 INFO [RS:0;61c2066a8a0f:33847 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c2066a8a0f%2C33847%2C1732084794617.meta:.meta(num 1732084798934) 2024-11-20T06:42:49,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741832_1008 (size=14178265) 2024-11-20T06:42:49,022 DEBUG [RS:0;61c2066a8a0f:33847 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/oldWALs 2024-11-20T06:42:49,022 INFO [RS:0;61c2066a8a0f:33847 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c2066a8a0f%2C33847%2C1732084794617:(num 1732084797919) 2024-11-20T06:42:49,022 DEBUG [RS:0;61c2066a8a0f:33847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:42:49,022 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:42:49,022 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T06:42:49,023 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T06:42:49,023 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T06:42:49,023 INFO [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T06:42:49,024 INFO [RS:0;61c2066a8a0f:33847 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33847 2024-11-20T06:42:49,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,33847,1732084794617 2024-11-20T06:42:49,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T06:42:49,038 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T06:42:49,047 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,33847,1732084794617] 2024-11-20T06:42:49,055 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,33847,1732084794617 already deleted, retry=false 2024-11-20T06:42:49,055 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,33847,1732084794617 expired; onlineServers=0 2024-11-20T06:42:49,055 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c2066a8a0f,33659,1732084793752' ***** 2024-11-20T06:42:49,055 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T06:42:49,056 INFO [M:0;61c2066a8a0f:33659 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T06:42:49,056 INFO [M:0;61c2066a8a0f:33659 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T06:42:49,056 DEBUG [M:0;61c2066a8a0f:33659 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T06:42:49,056 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T06:42:49,056 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732084797604 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732084797604,5,FailOnTimeoutGroup] 2024-11-20T06:42:49,056 DEBUG [M:0;61c2066a8a0f:33659 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T06:42:49,056 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732084797605 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732084797605,5,FailOnTimeoutGroup] 2024-11-20T06:42:49,056 INFO [M:0;61c2066a8a0f:33659 {}] hbase.ChoreService(370): Chore service for: master/61c2066a8a0f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T06:42:49,057 INFO [M:0;61c2066a8a0f:33659 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T06:42:49,057 DEBUG [M:0;61c2066a8a0f:33659 {}] master.HMaster(1795): Stopping service threads 2024-11-20T06:42:49,057 INFO [M:0;61c2066a8a0f:33659 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T06:42:49,057 INFO [M:0;61c2066a8a0f:33659 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T06:42:49,057 ERROR [M:0;61c2066a8a0f:33659 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (726517346) connection to localhost/127.0.0.1:36267 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36267,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-20T06:42:49,058 INFO [M:0;61c2066a8a0f:33659 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T06:42:49,058 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T06:42:49,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T06:42:49,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:42:49,063 DEBUG [M:0;61c2066a8a0f:33659 {}] zookeeper.ZKUtil(347): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T06:42:49,063 WARN [M:0;61c2066a8a0f:33659 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T06:42:49,064 INFO [M:0;61c2066a8a0f:33659 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/.lastflushedseqids 2024-11-20T06:42:49,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742450_1626 (size=72) 2024-11-20T06:42:49,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:42:49,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33847-0x101584211260001, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:42:49,147 INFO [RS:0;61c2066a8a0f:33847 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T06:42:49,148 INFO [RS:0;61c2066a8a0f:33847 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,33847,1732084794617; zookeeper connection closed. 2024-11-20T06:42:49,148 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a2afd9e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a2afd9e 2024-11-20T06:42:49,148 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T06:42:49,476 INFO [M:0;61c2066a8a0f:33659 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T06:42:49,476 INFO [M:0;61c2066a8a0f:33659 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T06:42:49,476 DEBUG [M:0;61c2066a8a0f:33659 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T06:42:49,476 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:42:49,476 DEBUG [M:0;61c2066a8a0f:33659 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:42:49,476 DEBUG [M:0;61c2066a8a0f:33659 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T06:42:49,476 DEBUG [M:0;61c2066a8a0f:33659 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:42:49,476 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=780.85 KB heapSize=961.50 KB 2024-11-20T06:42:49,491 DEBUG [M:0;61c2066a8a0f:33659 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9818966101d141a485e220ee17e5d1a7 is 82, key is hbase:meta,,1/info:regioninfo/1732084799015/Put/seqid=0 2024-11-20T06:42:49,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742451_1627 (size=5672) 2024-11-20T06:42:49,896 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2241 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9818966101d141a485e220ee17e5d1a7 2024-11-20T06:42:49,916 DEBUG [M:0;61c2066a8a0f:33659 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6cd4f478e5254fdba9b8ca0fae0fd0d9 is 2276, key is \x00\x00\x00\x00\x00\x00\x00%/proc:d/1732084829239/Put/seqid=0 2024-11-20T06:42:49,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742452_1628 (size=43851) 2024-11-20T06:42:50,319 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=780.30 KB at sequenceid=2241 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6cd4f478e5254fdba9b8ca0fae0fd0d9 2024-11-20T06:42:50,323 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6cd4f478e5254fdba9b8ca0fae0fd0d9 2024-11-20T06:42:50,340 DEBUG [M:0;61c2066a8a0f:33659 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67721c30cd544eeda371bd4f0b4a11f6 is 69, key is 61c2066a8a0f,33847,1732084794617/rs:state/1732084797659/Put/seqid=0 2024-11-20T06:42:50,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073742453_1629 (size=5156) 2024-11-20T06:42:50,743 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2241 (bloomFilter=true), to=hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67721c30cd544eeda371bd4f0b4a11f6 2024-11-20T06:42:50,747 DEBUG [M:0;61c2066a8a0f:33659 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9818966101d141a485e220ee17e5d1a7 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9818966101d141a485e220ee17e5d1a7 2024-11-20T06:42:50,750 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9818966101d141a485e220ee17e5d1a7, entries=8, sequenceid=2241, filesize=5.5 K 2024-11-20T06:42:50,750 DEBUG [M:0;61c2066a8a0f:33659 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6cd4f478e5254fdba9b8ca0fae0fd0d9 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6cd4f478e5254fdba9b8ca0fae0fd0d9 2024-11-20T06:42:50,753 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6cd4f478e5254fdba9b8ca0fae0fd0d9 2024-11-20T06:42:50,753 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6cd4f478e5254fdba9b8ca0fae0fd0d9, entries=168, sequenceid=2241, filesize=42.8 K 2024-11-20T06:42:50,754 DEBUG [M:0;61c2066a8a0f:33659 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67721c30cd544eeda371bd4f0b4a11f6 as hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/67721c30cd544eeda371bd4f0b4a11f6 2024-11-20T06:42:50,757 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36267/user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/67721c30cd544eeda371bd4f0b4a11f6, entries=1, sequenceid=2241, filesize=5.0 K 2024-11-20T06:42:50,758 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.HRegion(3140): Finished flush of dataSize ~780.85 KB/799592, heapSize ~961.20 KB/984272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1282ms, sequenceid=2241, compaction requested=false 2024-11-20T06:42:50,759 INFO [M:0;61c2066a8a0f:33659 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:42:50,759 DEBUG [M:0;61c2066a8a0f:33659 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732084969476Disabling compacts and flushes for region at 1732084969476Disabling writes for close at 1732084969476Obtaining lock to block concurrent updates at 1732084969476Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732084969476Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=799592, getHeapSize=984512, getOffHeapSize=0, getCellsCount=2434 at 1732084969477 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732084969477Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732084969478 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732084969491 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732084969491Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732084969900 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732084969915 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732084969915Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732084970323 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732084970339 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732084970339Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@641134ed: reopening flushed file at 1732084970747 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75b1fd1f: reopening flushed file at 1732084970750 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43986c64: reopening flushed file at 1732084970753 (+3 ms)Finished flush of dataSize ~780.85 KB/799592, heapSize ~961.20 KB/984272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1282ms, sequenceid=2241, compaction requested=false at 1732084970758 (+5 ms)Writing region close event to WAL at 1732084970759 (+1 ms)Closed at 1732084970759 2024-11-20T06:42:50,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40437 is added to blk_1073741830_1006 (size=946448) 2024-11-20T06:42:50,761 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0b157c72-0c4c-4a9e-c0b4-1153d27dc2f5/MasterData/WALs/61c2066a8a0f,33659,1732084793752/61c2066a8a0f%2C33659%2C1732084793752.1732084796850 not finished, retry = 0 2024-11-20T06:42:50,862 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T06:42:50,862 INFO [M:0;61c2066a8a0f:33659 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T06:42:50,862 INFO [M:0;61c2066a8a0f:33659 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33659 2024-11-20T06:42:50,863 INFO [M:0;61c2066a8a0f:33659 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T06:42:50,980 INFO [M:0;61c2066a8a0f:33659 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T06:42:50,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:42:50,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33659-0x101584211260000, quorum=127.0.0.1:52097, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:42:50,988 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T06:42:50,992 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T06:42:50,992 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T06:42:50,992 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T06:42:50,992 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/hadoop.log.dir/,STOPPED} 2024-11-20T06:42:50,994 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T06:42:50,994 WARN [BP-501673584-172.17.0.2-1732084790088 heartbeating to localhost/127.0.0.1:36267 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T06:42:50,994 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T06:42:50,994 WARN [BP-501673584-172.17.0.2-1732084790088 heartbeating to localhost/127.0.0.1:36267 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-501673584-172.17.0.2-1732084790088 (Datanode Uuid dae0415b-24a9-4464-8e5d-29d94cf63508) service to localhost/127.0.0.1:36267 2024-11-20T06:42:50,997 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/cluster_a8a03e82-e5a9-0028-3abc-1ed331460361/data/data1/current/BP-501673584-172.17.0.2-1732084790088 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T06:42:50,997 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/cluster_a8a03e82-e5a9-0028-3abc-1ed331460361/data/data2/current/BP-501673584-172.17.0.2-1732084790088 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T06:42:50,998 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T06:42:51,005 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T06:42:51,006 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T06:42:51,006 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T06:42:51,006 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T06:42:51,006 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8d065860-26e6-6152-7be3-787734b60d7c/hadoop.log.dir/,STOPPED} 2024-11-20T06:42:51,022 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T06:42:51,150 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down